From af22c2667fe5fdf9f717fe30db7fba1f9f3b609e Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Thu, 24 Jul 2025 16:56:54 -0400 Subject: [PATCH 01/15] feat(backup)_: local user data backups (#6722) Fixes https://github.com/status-im/status-desktop/issues/18248 https://github.com/status-im/status-desktop/issues/18107 https://github.com/status-im/status-desktop/issues/18120 https://github.com/status-im/status-desktop/issues/18122 https://github.com/status-im/status-desktop/issues/18141 https://github.com/status-im/status-desktop/issues/18121 Implements all the backend for the local user data backups. Uses the scaffold @osmaczko created to make it more extendable. Instead of using the old Messenger code for backups, we now use a new controller that let's other modules register to be backed up. This includes the messenger for chats, contacts and communities, the accounts service for settings and the wallet service for watch-only accounts. I also created new Protobufs for those backups, so that we no longer user the super generic Waku Backup protobuf. Finally, I added some integration tests and a functional test that does the whole flow. A lot of cleanups can still be done to reduce duplication, but they can be done in another commit/issue, since the internal of the services can be modified without the backup process being affected, since the protobufs are in place already. --- api/geth_backend.go | 8 + ...3221199_add_add_backup_path_setting.up.sql | 1 + mobile/status.go | 34 ++ multiaccounts/settings/columns.go | 5 + multiaccounts/settings/database.go | 29 +- .../settings/database_settings_manager.go | 1 + multiaccounts/settings/database_test.go | 25 + multiaccounts/settings/structs.go | 1 + node/backup/controller.go | 144 ++++++ node/backup/controller_test.go | 94 ++++ node/backup/core.go | 113 +++++ node/get_status_node.go | 87 +++- params/defaults.go | 1 + protocol/messenger.go | 7 + protocol/messenger_backup.go | 33 +- protocol/messenger_backup_handler.go | 57 ++- protocol/messenger_backup_test.go | 6 +- protocol/messenger_handler.go | 1 + protocol/messenger_local_backup.go | 70 +++ protocol/messenger_local_backup_test.go | 159 ++++++ protocol/messenger_sync_settings.go | 16 +- protocol/protobuf/accounts_local_backup.proto | 10 + .../protobuf/messenger_local_backup.proto | 16 + protocol/protobuf/service.go | 2 +- protocol/protobuf/wallet_local_backup.proto | 10 + protocol/requests/load_local_backup.go | 14 + services/accounts/service.go | 89 +++- services/accounts/settings.go | 5 + services/gif/gif_test.go | 14 + services/status/service.go | 4 + services/wallet/market/market_test.go | 21 +- services/wallet/service.go | 175 +++++++ tests-functional/README.MD | 37 +- tests-functional/clients/status_backend.py | 54 +- .../resources/images/test-image-200x200.jpg | Bin 0 -> 4925 bytes .../test_data/profile_showcase_utils.py | 93 ++++ .../test_data/test_old_user_data.bkp | Bin 0 -> 2622 bytes tests-functional/tests/test_local_backup.py | 466 ++++++++++++++++++ 38 files changed, 1835 insertions(+), 67 deletions(-) create mode 100644 appdatabase/migrations/sql/1753221199_add_add_backup_path_setting.up.sql create mode 100644 node/backup/controller.go create mode 100644 node/backup/controller_test.go create mode 100644 node/backup/core.go create mode 100644 protocol/messenger_local_backup.go create mode 100644 protocol/messenger_local_backup_test.go create mode 100644 protocol/protobuf/accounts_local_backup.proto create mode 100644 protocol/protobuf/messenger_local_backup.proto create mode 100644 protocol/protobuf/wallet_local_backup.proto create mode 100644 protocol/requests/load_local_backup.go create mode 100644 tests-functional/resources/images/test-image-200x200.jpg create mode 100644 tests-functional/resources/test_data/profile_showcase_utils.py create mode 100644 tests-functional/resources/test_data/test_old_user_data.bkp create mode 100644 tests-functional/tests/test_local_backup.py diff --git a/api/geth_backend.go b/api/geth_backend.go index 37ddfcfd545..ce9c8fd048d 100644 --- a/api/geth_backend.go +++ b/api/geth_backend.go @@ -838,6 +838,10 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error { } } + if err = b.statusNode.StartLocalBackup(); err != nil { + return err + } + err = b.multiaccountsDB.UpdateAccountTimestamp(acc.KeyUID, time.Now().Unix()) if err != nil { b.logger.Error("failed to update account") @@ -957,6 +961,10 @@ func (b *GethStatusBackend) startNodeWithAccount(acc multiaccounts.Account, pass } } + if err = b.statusNode.StartLocalBackup(); err != nil { + return err + } + err = b.multiaccountsDB.UpdateAccountTimestamp(acc.KeyUID, time.Now().Unix()) if err != nil { b.logger.Info("failed to update account") diff --git a/appdatabase/migrations/sql/1753221199_add_add_backup_path_setting.up.sql b/appdatabase/migrations/sql/1753221199_add_add_backup_path_setting.up.sql new file mode 100644 index 00000000000..258024ff2f3 --- /dev/null +++ b/appdatabase/migrations/sql/1753221199_add_add_backup_path_setting.up.sql @@ -0,0 +1 @@ +ALTER TABLE settings ADD COLUMN backup_path VARCHAR DEFAULT ''; diff --git a/mobile/status.go b/mobile/status.go index 75ab3807451..98d54698f56 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -2429,3 +2429,37 @@ func IntendedPanic(message string) string { panic(err) }) } + +func performLocalBackup() string { + filePath, err := statusBackend.StatusNode().PerformLocalBackup() + if err != nil { + return makeJSONResponse(err) + } + + respJSON, err := json.Marshal(map[string]interface{}{ + "filePath": filePath, + }) + if err != nil { + return makeJSONResponse(err) + } + + return string(respJSON) +} + +func PerformLocalBackup() string { + return callWithResponse(performLocalBackup) +} + +func LoadLocalBackup(requestJSON string) string { + var request requests.LoadLocalBackup + err := json.Unmarshal([]byte(requestJSON), &request) + if err != nil { + return makeJSONResponse(err) + } + err = request.Validate() + if err != nil { + return makeJSONResponse(err) + } + err = statusBackend.StatusNode().LoadLocalBackup(request.FilePath) + return makeJSONResponse(err) +} diff --git a/multiaccounts/settings/columns.go b/multiaccounts/settings/columns.go index 28cc8b93c1b..0656b1f1b15 100644 --- a/multiaccounts/settings/columns.go +++ b/multiaccounts/settings/columns.go @@ -34,6 +34,10 @@ var ( dBColumnName: "backup_fetched", valueHandler: BoolHandler, } + BackupPath = SettingField{ + reactFieldName: "backup-path", + dBColumnName: "backup_path", + } ChaosMode = SettingField{ reactFieldName: "chaos-mode?", dBColumnName: "chaos_mode", @@ -548,6 +552,7 @@ var ( AutoMessageEnabled, BackupEnabled, BackupFetched, + BackupPath, Bio, ChaosMode, CollectibleGroupByCollection, diff --git a/multiaccounts/settings/database.go b/multiaccounts/settings/database.go index 388ca9d5ae3..60d15047b34 100644 --- a/multiaccounts/settings/database.go +++ b/multiaccounts/settings/database.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "sync" "time" @@ -11,7 +12,7 @@ import ( "github.com/status-im/status-go/common/dbsetup" "github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/multiaccounts/errors" + maErrors "github.com/status-im/status-go/multiaccounts/errors" "github.com/status-im/status-go/nodecfg" "github.com/status-im/status-go/params" "github.com/status-im/status-go/sqlite" @@ -213,7 +214,7 @@ func (db *Database) getSettingFieldFromReactName(reactName string) (SettingField return s, nil } } - return SettingField{}, errors.ErrInvalidConfig + return SettingField{}, maErrors.ErrInvalidConfig } func (db *Database) makeSelectRow(setting SettingField) *sql.Row { @@ -243,12 +244,21 @@ func (db *Database) saveSetting(setting SettingField, value interface{}) error { return err } - _, err = update.Exec(value) + result, err := update.Exec(value) + if err != nil { + return err + } + rowsAffected, err := result.RowsAffected() if err != nil { return err } + if rowsAffected == 0 { + // If no rows were affected, it means the setting does not exist + return errors.New("settings not initialized, please call CreateSettings first") + } + if db.notifier != nil { db.notifier(setting, value) } @@ -334,7 +344,7 @@ func (db *Database) SaveSyncSetting(setting SettingField, value interface{}, clo return err } if clock <= ls { - return errors.ErrNewClockOlderThanCurrent + return maErrors.ErrNewClockOlderThanCurrent } err = db.SetSettingLastSynced(setting, clock) @@ -398,7 +408,7 @@ func (db *Database) GetSettings() (Settings, error) { test_networks_enabled, mutual_contact_enabled, profile_migration_needed, wallet_token_preferences_group_by_community, url_unfurling_mode, mnemonic_was_not_shown, wallet_show_community_asset_when_sending_tokens, wallet_display_assets_below_balance, wallet_display_assets_below_balance_threshold, wallet_collectible_preferences_group_by_collection, wallet_collectible_preferences_group_by_community, - peer_syncing_enabled, auto_refresh_tokens_enabled, last_tokens_update, news_feed_enabled, news_feed_last_fetched_timestamp, news_rss_enabled + peer_syncing_enabled, auto_refresh_tokens_enabled, last_tokens_update, news_feed_enabled, news_feed_last_fetched_timestamp, news_rss_enabled, backup_path FROM settings WHERE @@ -488,6 +498,7 @@ func (db *Database) GetSettings() (Settings, error) { &s.NewsFeedEnabled, &newsFeedLastFetchedTimestamp, &s.NewsRSSEnabled, + &s.BackupPath, ) if err != nil { @@ -929,3 +940,11 @@ func (db *Database) NewsRSSEnabled() (result bool, err error) { } return result, err } + +func (db *Database) BackupPath() (result string, err error) { + err = db.makeSelectRow(BackupPath).Scan(&result) + if err == sql.ErrNoRows { + return result, nil + } + return result, err +} diff --git a/multiaccounts/settings/database_settings_manager.go b/multiaccounts/settings/database_settings_manager.go index 1031d3bb770..01b8139c02f 100644 --- a/multiaccounts/settings/database_settings_manager.go +++ b/multiaccounts/settings/database_settings_manager.go @@ -63,6 +63,7 @@ type DatabaseSettingsManager interface { CanSyncOnMobileNetwork() (result bool, err error) ShouldBroadcastUserStatus() (result bool, err error) BackupEnabled() (result bool, err error) + BackupPath() (result string, err error) AutoMessageEnabled() (result bool, err error) LastBackup() (result uint64, err error) BackupFetched() (result bool, err error) diff --git a/multiaccounts/settings/database_test.go b/multiaccounts/settings/database_test.go index 6436f095642..a09ccc9fc87 100644 --- a/multiaccounts/settings/database_test.go +++ b/multiaccounts/settings/database_test.go @@ -2,10 +2,13 @@ package settings import ( "encoding/json" + "net/url" "os" "testing" "time" + "github.com/brianvoe/gofakeit/v6" + "github.com/stretchr/testify/require" "github.com/status-im/status-go/appdatabase" @@ -296,3 +299,25 @@ func TestDatabase_NewsRSSEnabled(t *testing.T) { require.NoError(t, err) require.Equal(t, false, settings.NewsRSSEnabled) } + +func TestDatabase_BackupPath(t *testing.T) { + db, stop := setupTestDB(t) + defer stop() + + require.NoError(t, db.CreateSettings(settings, config)) + + path, err := db.BackupPath() + require.NoError(t, err) + // The default backup path is empty + require.Equal(t, "", path) + + testPath, err := url.JoinPath(gofakeit.LetterN(3), gofakeit.LetterN(3)) + require.NoError(t, err) + require.NotEmpty(t, testPath) + err = db.SaveSetting(BackupPath.GetReactName(), testPath) + require.NoError(t, err) + + settings, err = db.GetSettings() + require.NoError(t, err) + require.Equal(t, testPath, settings.BackupPath) +} diff --git a/multiaccounts/settings/structs.go b/multiaccounts/settings/structs.go index 9a0218fe9ab..4b97f263da1 100644 --- a/multiaccounts/settings/structs.go +++ b/multiaccounts/settings/structs.go @@ -217,6 +217,7 @@ type Settings struct { LastBackup uint64 `json:"last-backup,omitempty"` BackupEnabled bool `json:"backup-enabled?,omitempty"` BackupFetched bool `json:"backup-fetched?,omitempty"` + BackupPath string `json:"backup-path,omitempty"` AutoMessageEnabled bool `json:"auto-message-enabled?,omitempty"` GifAPIKey string `json:"gifs/api-key"` TestNetworksEnabled bool `json:"test-networks-enabled?,omitempty"` diff --git a/node/backup/controller.go b/node/backup/controller.go new file mode 100644 index 00000000000..551a7dbefbd --- /dev/null +++ b/node/backup/controller.go @@ -0,0 +1,144 @@ +package backup + +import ( + "errors" + "os" + "path/filepath" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/status-im/status-go/common" +) + +type BackupConfig struct { + PrivateKey []byte + FileNameGetter func() (string, error) + BackupEnabled bool + Interval time.Duration +} + +type BackupProvider interface { + ExportBackup() ([]byte, error) + ImportBackup(data []byte) error +} + +type Controller struct { + config BackupConfig + core *core + logger *zap.Logger + quit chan struct{} + mutex sync.Mutex + wg *sync.WaitGroup +} + +func NewController(config BackupConfig, logger *zap.Logger) (*Controller, error) { + if len(config.PrivateKey) == 0 { + return nil, errors.New("private key must be provided") + } + if config.FileNameGetter == nil { + return nil, errors.New("filename getter must be provided") + } + + return &Controller{ + config: config, + core: newCore(), + logger: logger, + wg: &sync.WaitGroup{}, + quit: make(chan struct{}), + }, nil +} + +func (c *Controller) Register(componentName string, provider BackupProvider) { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.core.Register(componentName, provider) +} + +func (c *Controller) Start() { + if !c.config.BackupEnabled { + return + } + c.wg.Add(1) + + go func() { + defer common.LogOnPanic() + ticker := time.NewTicker(c.config.Interval) + defer ticker.Stop() + defer c.wg.Done() + for { + select { + case <-ticker.C: + _, err := c.PerformBackup() + if err != nil { + c.logger.Error("Error performing backup: %v\n", zap.Error(err)) + } + case <-c.quit: + return + } + } + }() +} + +func (c *Controller) Stop() { + close(c.quit) + c.wg.Wait() +} + +func (c *Controller) PerformBackup() (string, error) { + c.mutex.Lock() + defer c.mutex.Unlock() + + backupData, err := c.core.Create(c.config.PrivateKey) + if err != nil { + return "", err + } + + fileName, err := c.config.FileNameGetter() + if err != nil { + return "", err + } + + if err := os.MkdirAll(filepath.Dir(fileName), 0700); err != nil { + return "", err + } + + file, err := os.Create(fileName) + if err != nil { + return "", err + } + defer file.Close() + + _, err = file.Write(backupData) + if err != nil { + return "", err + } + + return fileName, nil +} + +func (c *Controller) LoadBackup(filePath string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + fileInfo, err := file.Stat() + if err != nil { + return err + } + + backupData := make([]byte, fileInfo.Size()) + _, err = file.Read(backupData) + if err != nil { + return err + } + + return c.core.Restore(c.config.PrivateKey, backupData) +} diff --git a/node/backup/controller_test.go b/node/backup/controller_test.go new file mode 100644 index 00000000000..7fd0ae4bdea --- /dev/null +++ b/node/backup/controller_test.go @@ -0,0 +1,94 @@ +package backup + +import ( + "encoding/json" + "encoding/xml" + "reflect" + "testing" + + "github.com/brianvoe/gofakeit/v6" + + "go.uber.org/zap" + + "github.com/stretchr/testify/require" +) + +type Foo struct { + Value int + PreciseValue float64 +} + +type Bar struct { + Names []string + Surname string +} + +type FooProvider struct { + foo Foo +} + +func (f FooProvider) ExportBackup() ([]byte, error) { + return json.Marshal(f.foo) +} + +var fooFromBackup Foo + +func (b FooProvider) ImportBackup(data []byte) error { + return json.Unmarshal(data, &fooFromBackup) +} + +type BarProvider struct { + bar Bar +} + +func (b BarProvider) ExportBackup() ([]byte, error) { + return xml.Marshal(b.bar) +} + +var barFromBackup Bar + +func (b BarProvider) ImportBackup(data []byte) error { + return xml.Unmarshal(data, &barFromBackup) +} + +func TestController(t *testing.T) { + logger, err := zap.NewDevelopment() + require.NoError(t, err) + filename := t.TempDir() + "/test_backup.bak" + controller, err := NewController(BackupConfig{ + FileNameGetter: func() (string, error) { return filename, nil }, + PrivateKey: []byte("0123456789abcdef0123456789abcdef"), + }, logger) + require.NoError(t, err) + + foo := Foo{} + bar := Bar{} + err = gofakeit.Struct(&foo) + require.NoError(t, err) + err = gofakeit.Struct(&bar) + require.NoError(t, err) + + fooProvider := FooProvider{ + foo: foo, + } + + barProvider := BarProvider{ + bar: bar, + } + + controller.Register("foo", fooProvider) + controller.Register("bar", barProvider) + + filename, err = controller.PerformBackup() + require.NoError(t, err) + require.Equal(t, filename, filename) + + require.False(t, reflect.DeepEqual(barProvider.bar, barFromBackup)) + require.False(t, reflect.DeepEqual(fooProvider.foo, fooFromBackup)) + + err = controller.LoadBackup(filename) + require.NoError(t, err) + + require.True(t, reflect.DeepEqual(barProvider.bar, barFromBackup)) + require.True(t, reflect.DeepEqual(fooProvider.foo, fooFromBackup)) +} diff --git a/node/backup/core.go b/node/backup/core.go new file mode 100644 index 00000000000..d608762c650 --- /dev/null +++ b/node/backup/core.go @@ -0,0 +1,113 @@ +package backup + +import ( + "bytes" + "encoding/gob" + "fmt" + + "github.com/status-im/status-go/eth-node/crypto" +) + +type core struct { + backupProviders map[string]BackupProvider +} + +func newCore() *core { + return &core{ + backupProviders: make(map[string]BackupProvider), + } +} + +func (c *core) Register( + componentName string, + provider BackupProvider, +) { + c.backupProviders[componentName] = provider +} + +func (b *core) Create(privateKey []byte) ([]byte, error) { + backup, err := b.exportBackup() + if err != nil { + return nil, fmt.Errorf("exportBackup failed: %w", err) + } + + data, err := marshal(backup) + if err != nil { + return nil, fmt.Errorf("marshal failed: %w", err) + } + + encryptedData, err := crypto.EncryptSymmetric(privateKey, data) + if err != nil { + return nil, fmt.Errorf("encrypt failed: %w", err) + } + + return encryptedData, nil +} + +func (b *core) Restore(privateKey []byte, encrypted []byte) error { + decrypted, err := crypto.DecryptSymmetric(privateKey, encrypted) + if err != nil { + return fmt.Errorf("decrypt failed: %w", err) + } + + data, err := unmarshal(decrypted) + if err != nil { + return fmt.Errorf("unmarshal failed: %w", err) + } + + err = b.importBackup(data) + if err != nil { + return fmt.Errorf("importBackup failed: %w", err) + } + + return nil +} + +func (b *core) exportBackup() (map[string][]byte, error) { + result := make(map[string][]byte, len(b.backupProviders)) + + for name, provider := range b.backupProviders { + raw, err := provider.ExportBackup() + if err != nil { + return nil, err + } + result[name] = raw + } + + return result, nil +} + +func (b *core) importBackup(data map[string][]byte) error { + for name, provider := range b.backupProviders { + raw, ok := data[name] + if !ok { + continue + } + if err := provider.ImportBackup(raw); err != nil { + return fmt.Errorf("importBackup %q failed: %w", name, err) + } + } + + return nil +} + +func marshal(data map[string][]byte) ([]byte, error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(data) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func unmarshal(data []byte) (map[string][]byte, error) { + buf := bytes.NewReader(data) + dec := gob.NewDecoder(buf) + var result map[string][]byte + err := dec.Decode(&result) + if err != nil { + return nil, err + } + return result, nil +} diff --git a/node/get_status_node.go b/node/get_status_node.go index f29da238200..7ed1d633732 100644 --- a/node/get_status_node.go +++ b/node/get_status_node.go @@ -4,9 +4,11 @@ import ( "context" "database/sql" "errors" + "fmt" "os" "path/filepath" "sync" + "time" "go.uber.org/zap" @@ -16,9 +18,13 @@ import ( "github.com/status-im/status-go/account" "github.com/status-im/status-go/connection" + "github.com/status-im/status-go/eth-node/crypto" "github.com/status-im/status-go/ipfs" "github.com/status-im/status-go/multiaccounts" + "github.com/status-im/status-go/node/backup" "github.com/status-im/status-go/params" + "github.com/status-im/status-go/pkg/pubsub" + "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/rpc" "github.com/status-im/status-go/server" accountssvc "github.com/status-im/status-go/services/accounts" @@ -113,10 +119,13 @@ type StatusNode struct { appGeneralSrvc *appgeneral.Service ethSrvc *eth.Service - accountsFeed event.Feed - walletFeed event.Feed - networksFeed event.Feed - settingsFeed event.Feed + accountsFeed event.Feed + walletFeed event.Feed + networksFeed event.Feed + settingsFeed event.Feed + accountsPublisher *pubsub.Publisher + + localBackup *backup.Controller } // New makes new instance of StatusNode. @@ -127,6 +136,7 @@ func New(transactor *transactions.Transactor, logger *zap.Logger) *StatusNode { transactor: transactor, logger: logger, publicMethods: make(map[string]bool), + accountsPublisher: pubsub.NewPublisher(), } } @@ -216,6 +226,70 @@ func (n *StatusNode) StartWithOptions(config *params.NodeConfig, options StartOp return n.startWithDB(config, options.AccountsManager) } +func (n *StatusNode) StartLocalBackup() error { + if n.localBackup != nil { + return errors.New("local backup already started") + } + + chatAccount, err := n.gethAccountsManager.SelectedChatAccount() + if err != nil { + return err + } + + privateKey := chatAccount.PrivateKey() + filenameGetter := func() (string, error) { + accountIdentifier := common.PubkeyToHex(&privateKey.PublicKey) + + backupPath, err := n.accountsSrvc.GetBackupPath() + if err != nil { + return "", err + } + var backupDir string + if backupPath != "" { + backupDir = backupPath + } else { + backupDir = filepath.Join(n.config.RootDataDir, "backups") + } + fullPath := filepath.Join(backupDir, fmt.Sprintf("%x_user_data.bkp", accountIdentifier[:4])) + return fullPath, nil + } + + n.localBackup, err = backup.NewController(backup.BackupConfig{ + PrivateKey: crypto.Keccak256(crypto.FromECDSA(privateKey)), + FileNameGetter: filenameGetter, + // TODO set to true to enable the local backup + BackupEnabled: false, + Interval: time.Minute * 30, + }, n.logger.Named("LocalBackup")) + if err != nil { + return err + } + + if n.accountsSrvc != nil { + n.localBackup.Register("settings", n.accountsSrvc) + } + + if n.walletSrvc != nil { + n.localBackup.Register("wallet", n.walletSrvc) + } + + if n.statusPublicSrvc != nil { + n.localBackup.Register("messenger", n.statusPublicSrvc.Messenger()) + } + + n.localBackup.Start() + + return nil +} + +func (n *StatusNode) PerformLocalBackup() (string, error) { + return n.localBackup.PerformBackup() +} + +func (n *StatusNode) LoadLocalBackup(filePath string) error { + return n.localBackup.LoadBackup(filePath) +} + func (n *StatusNode) SetMediaServerEnableTLS(enableTLS *bool) { n.mediaServerEnableTLS = enableTLS } @@ -304,6 +378,11 @@ func (n *StatusNode) Stop() error { return ErrNoRunningNode } + if n.localBackup != nil { + n.localBackup.Stop() + n.localBackup = nil + } + return n.stop() } diff --git a/params/defaults.go b/params/defaults.go index 047c205443f..51b4dead82c 100644 --- a/params/defaults.go +++ b/params/defaults.go @@ -8,6 +8,7 @@ const ( ArchivesRelativePath = "data/archivedata" TorrentTorrentsRelativePath = "data/torrents" + BackupsRelativePath = "backups" // SendTransactionMethodName https://docs.walletconnect.com/advanced/rpc-reference/ethereum-rpc#eth_sendtransaction SendTransactionMethodName = "eth_sendTransaction" diff --git a/protocol/messenger.go b/protocol/messenger.go index 7d640b1bf92..8cf4c83e43d 100644 --- a/protocol/messenger.go +++ b/protocol/messenger.go @@ -3524,6 +3524,13 @@ func (m *Messenger) saveDataAndPrepareResponse(messageState *ReceivedMessageStat if ok { contactsToSave = append(contactsToSave, contact) messageState.Response.AddContact(contact) + + _, ok := m.allContacts.Load(id) + if !ok { + // If the contact is not in the allContacts map, it means it's a new contact + // and we need to add it to the allContacts map. + m.allContacts.Store(id, contact) + } } return true }) diff --git a/protocol/messenger_backup.go b/protocol/messenger_backup.go index 1be48224d10..c3f4365f774 100644 --- a/protocol/messenger_backup.go +++ b/protocol/messenger_backup.go @@ -96,17 +96,15 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) { return 0, err } chatsToBackup := m.backupChats(ctx, clock) - if err != nil { - return 0, err - } + profileToBackup, err := m.backupProfile(ctx, clock) if err != nil { return 0, err } - _, settings, errors := m.prepareSyncSettingsMessages(clock, true) - if len(errors) != 0 { + _, settings, err := m.prepareSyncSettingsMessages(clock, true) + if err != nil { // return just the first error, the others have been logged - return 0, errors[0] + return 0, err } keypairsToBackup, err := m.backupKeypairs() @@ -136,7 +134,7 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) { }, ProfileDetails: &protobuf.FetchingBackedUpDataDetails{ DataNumber: uint32(0), - TotalNumber: uint32(len(profileToBackup)), + TotalNumber: uint32(1), // Profile is always one }, SettingsDetails: &protobuf.FetchingBackedUpDataDetails{ DataNumber: uint32(0), @@ -176,14 +174,12 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) { } // Update profile messages encode and dispatch - for i, d := range profileToBackup { - pb := backupDetailsOnly() - pb.ProfileDetails.DataNumber = uint32(i + 1) - pb.Profile = d.Profile - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } + pb := backupDetailsOnly() + pb.ProfileDetails.DataNumber = uint32(1) + pb.Profile = profileToBackup.Profile + err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) + if err != nil { + return 0, err } // Update chats encode and dispatch @@ -208,6 +204,7 @@ func (m *Messenger) BackupData(ctx context.Context) (uint64, error) { } } + // TODO get rid of keypairs // Update keypairs messages encode and dispatch for i, d := range keypairsToBackup { pb := backupDetailsOnly() @@ -447,7 +444,7 @@ func (m *Messenger) buildSyncContactMessage(contact *Contact) *protobuf.SyncInst } } -func (m *Messenger) backupProfile(ctx context.Context, clock uint64) ([]*protobuf.Backup, error) { +func (m *Messenger) backupProfile(ctx context.Context, clock uint64) (*protobuf.Backup, error) { displayName, err := m.settings.DisplayName() if err != nil { return nil, err @@ -515,9 +512,7 @@ func (m *Messenger) backupProfile(ctx context.Context, clock uint64) ([]*protobu }, } - backupMessages := []*protobuf.Backup{backupMessage} - - return backupMessages, nil + return backupMessage, nil } func (m *Messenger) backupKeypairs() ([]*protobuf.Backup, error) { diff --git a/protocol/messenger_backup_handler.go b/protocol/messenger_backup_handler.go index b217fa9b240..580d005f736 100644 --- a/protocol/messenger_backup_handler.go +++ b/protocol/messenger_backup_handler.go @@ -60,6 +60,7 @@ func (m *Messenger) HandleBackup(state *ReceivedMessageState, message *protobuf. return nil } +// TODO remove this function once we do the Waku backup removal func (m *Messenger) handleBackup(state *ReceivedMessageState, message *protobuf.Backup) []error { var errors []error @@ -85,7 +86,7 @@ func (m *Messenger) handleBackup(state *ReceivedMessageState, message *protobuf. errors = append(errors, communityErrors...) } - err = m.handleBackedUpSettings(message.Setting) + err = m.HandleBackedUpSettings(message.Setting) if err != nil { errors = append(errors, err) } @@ -95,7 +96,7 @@ func (m *Messenger) handleBackup(state *ReceivedMessageState, message *protobuf. errors = append(errors, err) } - err = m.handleWatchOnlyAccount(message.WatchOnlyAccount) + err = m.HandleWatchOnlyAccount(message.WatchOnlyAccount) if err != nil { errors = append(errors, err) } @@ -126,6 +127,34 @@ func (m *Messenger) handleBackup(state *ReceivedMessageState, message *protobuf. return errors } +func (m *Messenger) handleLocalBackup(state *ReceivedMessageState, message *protobuf.MessengerLocalBackup) []error { + var errors []error + + err := m.handleBackedUpProfile(message.Profile, message.Clock) + if err != nil { + errors = append(errors, err) + } + + for _, contact := range message.Contacts { + err = m.HandleSyncInstallationContactV2(state, contact, nil) + if err != nil { + errors = append(errors, err) + } + } + + err = m.handleSyncChats(state, message.Chats) + if err != nil { + errors = append(errors, err) + } + + communityErrors := m.handleLocalBackupCommunities(state, message.Communities) + if len(communityErrors) > 0 { + errors = append(errors, communityErrors...) + } + + return errors +} + func (m *Messenger) updateBackupFetchProgress(message *protobuf.Backup, response *wakusync.WakuBackedUpDataResponse, state *ReceivedMessageState) error { if m.backedUpFetchingStatus == nil { return nil @@ -376,7 +405,7 @@ func (m *Messenger) handleBackedUpProfile(message *protobuf.BackedUpProfile, bac return err } -func (m *Messenger) handleBackedUpSettings(message *protobuf.SyncSetting) error { +func (m *Messenger) HandleBackedUpSettings(message *protobuf.SyncSetting) error { if message == nil { return nil } @@ -405,7 +434,7 @@ func (m *Messenger) handleBackedUpSettings(message *protobuf.SyncSetting) error m.account.Name = message.GetValueString() err = m.multiAccounts.SaveAccount(*m.account) if err != nil { - m.logger.Warn("[handleBackedUpSettings] failed to save account", zap.Error(err)) + m.logger.Warn("[HandleBackedUpSettings] failed to save account", zap.Error(err)) return nil } } @@ -456,7 +485,7 @@ func (m *Messenger) handleKeypair(message *protobuf.SyncKeypair) error { return nil } -func (m *Messenger) handleWatchOnlyAccount(message *protobuf.SyncAccount) error { +func (m *Messenger) HandleWatchOnlyAccount(message *protobuf.SyncAccount) error { if message == nil { return nil } @@ -492,6 +521,7 @@ func syncInstallationCommunitiesSet(communities []*protobuf.SyncInstallationComm return ret } +// TODO remove this function once we do the Waku backup removal func (m *Messenger) handleSyncedCommunities(state *ReceivedMessageState, message *protobuf.Backup) []error { var errors []error for _, syncCommunity := range syncInstallationCommunitiesSet(message.Communities) { @@ -509,6 +539,23 @@ func (m *Messenger) handleSyncedCommunities(state *ReceivedMessageState, message return errors } +func (m *Messenger) handleLocalBackupCommunities(state *ReceivedMessageState, communities []*protobuf.SyncInstallationCommunity) []error { + var errors []error + for _, syncCommunity := range syncInstallationCommunitiesSet(communities) { + err := m.handleSyncInstallationCommunity(state, syncCommunity) + if err != nil { + errors = append(errors, err) + } + + err = m.requestCommunityKeysAndSharedAddresses(state, syncCommunity) + if err != nil { + errors = append(errors, err) + } + } + + return errors +} + func (m *Messenger) requestCommunityKeysAndSharedAddresses(state *ReceivedMessageState, syncCommunity *protobuf.SyncInstallationCommunity) error { if !syncCommunity.Joined { return nil diff --git a/protocol/messenger_backup_test.go b/protocol/messenger_backup_test.go index 6067a99a310..8bbef814228 100644 --- a/protocol/messenger_backup_test.go +++ b/protocol/messenger_backup_test.go @@ -44,7 +44,6 @@ func (s *MessengerBackupSuite) TestBackupContacts() { defer TearDownMessenger(&s.Suite, bob2) // Create 2 contacts - contact1Key, err := crypto.GenerateKey() s.Require().NoError(err) contactID1 := types.EncodeHex(crypto.FromECDSAPub(&contact1Key.PublicKey)) @@ -61,6 +60,7 @@ func (s *MessengerBackupSuite) TestBackupContacts() { s.Require().Len(bob1.Contacts(), 2) + // Validate contacts actualContacts := bob1.Contacts() if actualContacts[0].ID == contactID1 { s.Require().Equal(actualContacts[0].ID, contactID1) @@ -76,7 +76,6 @@ func (s *MessengerBackupSuite) TestBackupContacts() { s.Require().True(actualContacts[1].added()) // Backup - clock, err := bob1.BackupData(context.Background()) s.Require().NoError(err) @@ -825,8 +824,7 @@ func (s *MessengerBackupSuite) TestBackupCommunities() { s.Require().NoError(err) defer TearDownMessenger(&s.Suite, bob2) - // Create a communitie - + // Create a community description := &requests.CreateCommunity{ Membership: protobuf.CommunityPermissions_AUTO_ACCEPT, Name: "status", diff --git a/protocol/messenger_handler.go b/protocol/messenger_handler.go index c6b228978ff..89fc917e835 100644 --- a/protocol/messenger_handler.go +++ b/protocol/messenger_handler.go @@ -579,6 +579,7 @@ func (m *Messenger) handleSyncChats(messageState *ReceivedMessageState, chats [] if err != nil { return err } + messageState.AllChats.Store(chat.ID, chat) messageState.Response.AddChat(chat) } diff --git a/protocol/messenger_local_backup.go b/protocol/messenger_local_backup.go new file mode 100644 index 00000000000..26532c8ddbe --- /dev/null +++ b/protocol/messenger_local_backup.go @@ -0,0 +1,70 @@ +package protocol + +import ( + "errors" + + "github.com/golang/protobuf/proto" + + "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/signal" +) + +func (m *Messenger) ExportBackup() ([]byte, error) { + backup := &protobuf.MessengerLocalBackup{} + + clock, _ := m.getLastClockWithRelatedChat() + contactsToBackup := m.backupContacts(m.ctx) + communitiesToBackup, err := m.backupCommunities(m.ctx, clock) + if err != nil { + return nil, err + } + chatsToBackup := m.backupChats(m.ctx, clock) + profileToBackup, err := m.backupProfile(m.ctx, clock) + if err != nil { + return nil, err + } + + for _, d := range contactsToBackup { + backup.Contacts = append(backup.Contacts, d.Contacts...) + } + for _, d := range communitiesToBackup { + backup.Communities = append(backup.Communities, d.Communities...) + } + backup.Profile = profileToBackup.Profile + for _, d := range chatsToBackup { + backup.Chats = append(backup.Chats, d.Chats...) + } + return proto.Marshal(backup) +} + +func (m *Messenger) ImportBackup(data []byte) error { + var backup protobuf.MessengerLocalBackup + err := proto.Unmarshal(data, &backup) + if err != nil { + return err + } + + state := ReceivedMessageState{ + Response: &MessengerResponse{}, + AllChats: &chatMap{}, + AllContacts: &contactMap{ + me: m.selfContact, + }, + Timesource: m.getTimesource(), + ModifiedContacts: &stringBoolMap{}, + ModifiedInstallations: &stringBoolMap{}, + } + errs := m.handleLocalBackup( + &state, + &backup, + ) + if len(errs) > 0 { + return errors.Join(errs...) + } + + response, err := m.saveDataAndPrepareResponse(&state) + + signal.SendNewMessages(response) + + return err +} diff --git a/protocol/messenger_local_backup_test.go b/protocol/messenger_local_backup_test.go new file mode 100644 index 00000000000..3c565e85597 --- /dev/null +++ b/protocol/messenger_local_backup_test.go @@ -0,0 +1,159 @@ +package protocol + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/status-im/status-go/eth-node/crypto" + "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/protocol/requests" +) + +func TestMessengerLocalBackupSuite(t *testing.T) { + suite.Run(t, new(MessengerLocalBackupSuite)) +} + +type MessengerLocalBackupSuite struct { + MessengerBaseTestSuite +} + +func (s *MessengerLocalBackupSuite) TestLocalBackup() { + // Create bob1 + bob1 := s.anotherMessenger() + defer TearDownMessenger(&s.Suite, bob1) + + // Create bob2 + bob2 := s.anotherMessenger() + defer TearDownMessenger(&s.Suite, bob2) + + // -------------------- CONTACTS -------------------- + // Create 2 contacts + contact1Key, err := crypto.GenerateKey() + s.Require().NoError(err) + contactID1 := types.EncodeHex(crypto.FromECDSAPub(&contact1Key.PublicKey)) + + _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID1}) + s.Require().NoError(err) + + contact2Key, err := crypto.GenerateKey() + s.Require().NoError(err) + contactID2 := types.EncodeHex(crypto.FromECDSAPub(&contact2Key.PublicKey)) + + _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID2}) + s.Require().NoError(err) + + s.Require().Len(bob1.Contacts(), 2) + + // Validate contacts on bob1 + actualContacts := bob1.Contacts() + if actualContacts[0].ID == contactID1 { + s.Require().Equal(actualContacts[0].ID, contactID1) + s.Require().Equal(actualContacts[1].ID, contactID2) + } else { + s.Require().Equal(actualContacts[0].ID, contactID2) + s.Require().Equal(actualContacts[1].ID, contactID1) + } + s.Require().Equal(ContactRequestStateSent, actualContacts[0].ContactRequestLocalState) + s.Require().Equal(ContactRequestStateSent, actualContacts[1].ContactRequestLocalState) + s.Require().True(actualContacts[0].added()) + s.Require().True(actualContacts[1].added()) + + // Check that bob2 has no contacts + s.Require().Len(bob2.Contacts(), 0) + + //-------------------- COMMUNITIES -------------------- + // Create a community + description := &requests.CreateCommunity{ + Membership: protobuf.CommunityPermissions_AUTO_ACCEPT, + Name: "status", + Color: "#ffffff", + Description: "status community description", + } + response, err := bob1.CreateCommunity(description, true) + s.Require().NoError(err) + s.Require().NotNil(response) + s.Require().Len(response.Communities(), 1) + + // Check bob2 + communities, err := bob2.Communities() + s.Require().NoError(err) + s.Require().Len(communities, 0) + + // --------------------- LEFT COMMUNITY -------------------- + // Create another community + description = &requests.CreateCommunity{ + Membership: protobuf.CommunityPermissions_MANUAL_ACCEPT, + Name: "other-status", + Color: "#fffff4", + Description: "other status community description", + } + + response, err = bob1.CreateCommunity(description, true) + s.Require().NoError(err) + s.Require().NotNil(response) + s.Require().Len(response.Communities(), 1) + + newCommunity := response.Communities()[0] + + // Leave community + response, err = bob1.LeaveCommunity(newCommunity.ID()) + s.Require().NoError(err) + s.Require().NotNil(response) + + // Check bob2 + communities, err = bob2.Communities() + s.Require().NoError(err) + s.Require().Len(communities, 0) + + // --------------------- CHATS -------------------- + // Create a group chat + response, err = bob1.CreateGroupChatWithMembers(context.Background(), "group", []string{}) + s.NoError(err) + s.Require().Len(response.Chats(), 1) + + ourGroupChat := response.Chats()[0] + + err = bob1.SaveChat(ourGroupChat) + s.NoError(err) + + // Create a one-to-one chat + alice := s.newMessenger() + defer TearDownMessenger(&s.Suite, alice) + + ourOneOneChat := CreateOneToOneChat("Our 1TO1", &alice.identity.PublicKey, alice.getTimesource()) + err = bob1.SaveChat(ourOneOneChat) + s.Require().NoError(err) + + // -------------------- BACKUP -------------------- + // Backup + marshalledBackup, err := bob1.ExportBackup() + s.Require().NoError(err) + + // Import the backup file and process it + err = bob2.ImportBackup(marshalledBackup) + s.Require().NoError(err) + + // -------------------- VALIDATE BACKUP -------------------- + // Validate contacts on bob2 + s.Require().Len(bob2.Contacts(), 2) + + // Validate communities on bob2 + communities, err = bob2.JoinedCommunities() + s.Require().NoError(err) + s.Require().Len(communities, 1) + + // Validate chats on bob2 + // Group chat + chat, ok := bob2.allChats.Load(ourGroupChat.ID) + s.Require().True(ok) + s.Require().Equal(ourGroupChat.Name, chat.Name) + + // One on one chat + chat, ok = bob2.allChats.Load(ourOneOneChat.ID) + s.Require().True(ok) + s.Require().True(chat.Active) + s.Require().Equal("", chat.Name) +} diff --git a/protocol/messenger_sync_settings.go b/protocol/messenger_sync_settings.go index 727d1199e09..ab4413e35f7 100644 --- a/protocol/messenger_sync_settings.go +++ b/protocol/messenger_sync_settings.go @@ -3,6 +3,7 @@ package protocol import ( "context" "encoding/json" + errorsLib "errors" "go.uber.org/zap" @@ -14,10 +15,11 @@ import ( ) // syncSettings syncs all settings that are syncable -func (m *Messenger) prepareSyncSettingsMessages(currentClock uint64, prepareForBackup bool) (resultRaw []*common.RawMessage, resultSync []*protobuf.SyncSetting, errors []error) { +func (m *Messenger) prepareSyncSettingsMessages(currentClock uint64, prepareForBackup bool) (resultRaw []*common.RawMessage, resultSync []*protobuf.SyncSetting, errorResult error) { + var errors []error s, err := m.settings.GetSettings() if err != nil { - errors = append(errors, err) + errorResult = err return } @@ -36,7 +38,7 @@ func (m *Messenger) prepareSyncSettingsMessages(currentClock uint64, prepareForB clock, err := m.settings.GetSettingLastSynced(sf) if err != nil { logger.Error("m.settings.GetSettingLastSynced", zap.Error(err), zap.String("SettingField", sf.GetDBName())) - errors = append(errors, err) + errorResult = err return } if clock == 0 { @@ -55,6 +57,7 @@ func (m *Messenger) prepareSyncSettingsMessages(currentClock uint64, prepareForB resultSync = append(resultSync, sm) } } + errorResult = errorsLib.Join(errors...) return } @@ -62,11 +65,10 @@ func (m *Messenger) syncSettings(rawMessageHandler RawMessageHandler) error { logger := m.logger.Named("syncSettings") clock, _ := m.getLastClockWithRelatedChat() - rawMessages, _, errors := m.prepareSyncSettingsMessages(clock, false) + rawMessages, _, err := m.prepareSyncSettingsMessages(clock, false) - if len(errors) != 0 { - // return just the first error, the others have been logged - return errors[0] + if err != nil { + return err } for _, rm := range rawMessages { diff --git a/protocol/protobuf/accounts_local_backup.proto b/protocol/protobuf/accounts_local_backup.proto new file mode 100644 index 00000000000..b09126662ef --- /dev/null +++ b/protocol/protobuf/accounts_local_backup.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +import "sync_settings.proto"; + +option go_package = "./;protobuf"; +package protobuf; + +message AccountsLocalBackup { + repeated SyncSetting settings = 1; +} diff --git a/protocol/protobuf/messenger_local_backup.proto b/protocol/protobuf/messenger_local_backup.proto new file mode 100644 index 00000000000..a8975a3e5e1 --- /dev/null +++ b/protocol/protobuf/messenger_local_backup.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "pairing.proto"; + +option go_package = "./;protobuf"; +package protobuf; + + +message MessengerLocalBackup { + uint64 clock = 1; + + repeated SyncInstallationContactV2 contacts = 2; + repeated SyncInstallationCommunity communities = 3; + repeated SyncChat chats = 4; + BackedUpProfile profile = 5; +} \ No newline at end of file diff --git a/protocol/protobuf/service.go b/protocol/protobuf/service.go index e3052c2ea80..b1adc62c3cd 100644 --- a/protocol/protobuf/service.go +++ b/protocol/protobuf/service.go @@ -4,7 +4,7 @@ import ( "github.com/golang/protobuf/proto" ) -//go:generate protoc --go_out=. ./chat_message.proto ./application_metadata_message.proto ./membership_update_message.proto ./command.proto ./contact.proto ./pairing.proto ./push_notifications.proto ./emoji_reaction.proto ./enums.proto ./shard.proto ./group_chat_invitation.proto ./chat_identity.proto ./communities.proto ./pin_message.proto ./anon_metrics.proto ./status_update.proto ./sync_settings.proto ./contact_verification.proto ./community_update.proto ./community_shard_key.proto ./url_data.proto ./community_privileged_user_sync_message.proto ./profile_showcase.proto ./segment_message.proto +//go:generate protoc --go_out=. ./chat_message.proto ./application_metadata_message.proto ./membership_update_message.proto ./command.proto ./contact.proto ./pairing.proto ./push_notifications.proto ./emoji_reaction.proto ./enums.proto ./shard.proto ./group_chat_invitation.proto ./chat_identity.proto ./communities.proto ./pin_message.proto ./anon_metrics.proto ./status_update.proto ./sync_settings.proto ./contact_verification.proto ./community_update.proto ./community_shard_key.proto ./url_data.proto ./community_privileged_user_sync_message.proto ./profile_showcase.proto ./segment_message.proto ./messenger_local_backup.proto ./wallet_local_backup.proto ./accounts_local_backup.proto func Unmarshal(payload []byte) (*ApplicationMetadataMessage, error) { var message ApplicationMetadataMessage diff --git a/protocol/protobuf/wallet_local_backup.proto b/protocol/protobuf/wallet_local_backup.proto new file mode 100644 index 00000000000..7d3cd753bec --- /dev/null +++ b/protocol/protobuf/wallet_local_backup.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +import "pairing.proto"; + +option go_package = "./;protobuf"; +package protobuf; + +message WalletLocalBackup { + repeated SyncAccount watchOnlyAccounts = 1; +} diff --git a/protocol/requests/load_local_backup.go b/protocol/requests/load_local_backup.go new file mode 100644 index 00000000000..2384f6cffa2 --- /dev/null +++ b/protocol/requests/load_local_backup.go @@ -0,0 +1,14 @@ +package requests + +import "errors" + +type LoadLocalBackup struct { + FilePath string `json:"filePath"` +} + +func (c *LoadLocalBackup) Validate() error { + if c.FilePath == "" { + return errors.New("filePath must be provided") + } + return nil +} diff --git a/services/accounts/service.go b/services/accounts/service.go index 57e2b9700b2..514b0479be0 100644 --- a/services/accounts/service.go +++ b/services/accounts/service.go @@ -1,16 +1,24 @@ package accounts import ( + "errors" + "time" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" - "github.com/status-im/status-go/multiaccounts/settings" - "github.com/status-im/status-go/server" + + "github.com/golang/protobuf/proto" "github.com/status-im/status-go/account" + "github.com/status-im/status-go/eth-node/crypto" + "github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/multiaccounts" "github.com/status-im/status-go/multiaccounts/accounts" + "github.com/status-im/status-go/multiaccounts/settings" "github.com/status-im/status-go/params" "github.com/status-im/status-go/protocol" + "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/server" ) // NewService initializes service instance. @@ -18,7 +26,7 @@ func NewService(db *accounts.Database, mdb *multiaccounts.Database, manager *acc return &Service{db, mdb, manager, config, feed, nil, mediaServer} } -// Service is a browsers service. +// Service is an accounts service. type Service struct { db *accounts.Database mdb *multiaccounts.Database @@ -77,6 +85,10 @@ func (s *Service) GetSettings() (settings.Settings, error) { return s.db.GetSettings() } +func (s *Service) GetBackupPath() (string, error) { + return s.db.BackupPath() +} + func (s *Service) GetMessenger() *protocol.Messenger { return s.messenger } @@ -89,3 +101,74 @@ func (s *Service) VerifyPassword(password string) bool { _, err = s.manager.VerifyAccountPassword(s.config.KeyStoreDir, address.Hex(), password) return err == nil } + +func (s *Service) prepareSyncSettingsMessages(currentClock uint64, prepareForBackup bool) (resultSync []*protobuf.SyncSetting, errorResult error) { + var errs []error + dbSettings, err := s.db.GetSettings() + if err != nil { + errorResult = err + return + } + + for _, sf := range settings.SettingFieldRegister { + if !sf.CanSync(settings.FromStruct) { + continue + } + + // DisplayName is backed up via `protobuf.BackedUpProfile` message. + if prepareForBackup && sf.SyncProtobufFactory().SyncSettingProtobufType() == protobuf.SyncSetting_DISPLAY_NAME { + continue + } + + // Pull clock from the db + clock, err := s.db.GetSettingLastSynced(sf) + if err != nil { + errorResult = err + return + } + if clock == 0 { + clock = currentClock + } + + // Build protobuf + _, sm, err := sf.SyncProtobufFactory().FromStruct()(dbSettings, clock, types.EncodeHex(crypto.FromECDSAPub(s.messenger.IdentityPublicKey()))) + if err != nil { + // Collect errors to give other sync messages a chance to send + errs = append(errs, err) + } + + resultSync = append(resultSync, sm) + } + errorResult = errors.Join(errs...) + return +} + +func (s *Service) ExportBackup() ([]byte, error) { + backup := &protobuf.AccountsLocalBackup{} + + settings, err := s.prepareSyncSettingsMessages(uint64(time.Now().UnixMilli()), true) + if err != nil { + return nil, err + } + backup.Settings = append(backup.Settings, settings...) + + return proto.Marshal(backup) +} + +func (s *Service) ImportBackup(data []byte) error { + var backup protobuf.AccountsLocalBackup + err := proto.Unmarshal(data, &backup) + if err != nil { + return err + } + var errs []error + + for _, setting := range backup.Settings { + // TODO is it ok to use the messenger here? Otherwise, I have to duplicate a lot of code + err = s.messenger.HandleBackedUpSettings(setting) + if err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} diff --git a/services/accounts/settings.go b/services/accounts/settings.go index cbe3f67e849..075dfafc5c7 100644 --- a/services/accounts/settings.go +++ b/services/accounts/settings.go @@ -63,6 +63,11 @@ func (api *SettingsAPI) NewsRSSEnabled() (bool, error) { return api.db.NewsRSSEnabled() } +// Backup Settings +func (api *SettingsAPI) BackupPath() (string, error) { + return api.db.BackupPath() +} + // Notifications Settings func (api *SettingsAPI) NotificationsGetAllowNotifications() (bool, error) { return api.db.GetAllowNotifications() diff --git a/services/gif/gif_test.go b/services/gif/gif_test.go index 61c72482c4d..2c824218ad8 100644 --- a/services/gif/gif_test.go +++ b/services/gif/gif_test.go @@ -9,6 +9,8 @@ import ( "github.com/status-im/status-go/appdatabase" "github.com/status-im/status-go/multiaccounts/accounts" + "github.com/status-im/status-go/multiaccounts/settings" + "github.com/status-im/status-go/params" "github.com/status-im/status-go/t/helpers" ) @@ -21,6 +23,18 @@ func setupSQLTestDb(t *testing.T) (*sql.DB, func()) { func setupTestDB(t *testing.T, db *sql.DB) (*accounts.Database, func()) { acc, err := accounts.NewDB(db) require.NoError(t, err) + config := params.NodeConfig{ + NetworkID: 10, + DataDir: "test", + } + networks := json.RawMessage("{}") + settingsObj := settings.Settings{ + Networks: &networks, + } + + err = acc.CreateSettings(settingsObj, config) + require.NoError(t, err) + return acc, func() { require.NoError(t, db.Close()) } diff --git a/services/status/service.go b/services/status/service.go index 4fe44ef4941..141a63134c3 100644 --- a/services/status/service.go +++ b/services/status/service.go @@ -42,6 +42,10 @@ func (s *Service) APIs() []rpc.API { } } +func (s *Service) Messenger() *protocol.Messenger { + return s.messenger +} + // NewPublicAPI returns a reference to the PublicAPI object func NewPublicAPI(s *Service) *PublicAPI { api := &PublicAPI{ diff --git a/services/wallet/market/market_test.go b/services/wallet/market/market_test.go index 43a3a96b56a..5a338d7286d 100644 --- a/services/wallet/market/market_test.go +++ b/services/wallet/market/market_test.go @@ -2,6 +2,7 @@ package market import ( "context" + "encoding/json" "errors" "testing" @@ -12,6 +13,9 @@ import ( "github.com/stretchr/testify/require" "github.com/status-im/status-go/appdatabase" + "github.com/status-im/status-go/multiaccounts/accounts" + "github.com/status-im/status-go/multiaccounts/settings" + "github.com/status-im/status-go/params" "github.com/status-im/status-go/rpc/network" mock_market "github.com/status-im/status-go/services/wallet/market/mock" "github.com/status-im/status-go/services/wallet/thirdparty" @@ -30,10 +34,25 @@ func setupTokenManager(t *testing.T) (*token.Manager, func()) { nm := network.NewManager(appDb, nil, nil, nil) - return token.NewTokenManager(walletDb, nil, nil, nm, appDb, nil, nil, nil, nil, token.NewPersistence(walletDb)), + accDb, err := accounts.NewDB(appDb) + require.NoError(t, err) + config := params.NodeConfig{ + NetworkID: 10, + DataDir: "test", + } + networks := json.RawMessage("{}") + settingsObj := settings.Settings{ + Networks: &networks, + } + + err = accDb.CreateSettings(settingsObj, config) + require.NoError(t, err) + + return token.NewTokenManager(walletDb, nil, nil, nm, appDb, nil, nil, nil, accDb, token.NewPersistence(walletDb)), func() { require.NoError(t, appDb.Close()) require.NoError(t, walletDb.Close()) + require.NoError(t, accDb.Close()) } } diff --git a/services/wallet/service.go b/services/wallet/service.go index 1c00cff7f0f..e24e930f72b 100644 --- a/services/wallet/service.go +++ b/services/wallet/service.go @@ -4,10 +4,14 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "sync" "time" + "github.com/golang/protobuf/proto" + + "github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/services/wallet/thirdparty/market/cryptocompare" "github.com/ethereum/go-ethereum/common" @@ -17,8 +21,11 @@ import ( "github.com/status-im/status-go/account" "github.com/status-im/status-go/logutils" "github.com/status-im/status-go/multiaccounts/accounts" + multiaccountscommon "github.com/status-im/status-go/multiaccounts/common" "github.com/status-im/status-go/params" protocolCommon "github.com/status-im/status-go/protocol/common" + "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/protocol/wakusync" "github.com/status-im/status-go/rpc" "github.com/status-im/status-go/server" "github.com/status-im/status-go/services/ens/ensresolver" @@ -53,6 +60,16 @@ const ( defaultAutoRefreshCheckInterval = 3 * time.Minute // interval after which we should check if we should trigger the auto-refresh ) +// TODO this is duplicated +var ( + ErrNotWatchOnlyAccount = errors.New("an account is not a watch only account") + ErrTryingToStoreOldWalletAccount = errors.New("trying to store an old wallet account") +) + +const ( + EventWatchOnlyAccountRetrieved walletevent.EventType = "wallet-watch-only-account-retrieved" +) + func createCoingeckoProxyClient(config params.MarketDataProxyConfig) *coingecko.Client { baseURL := leaderboard.GetMarketProxyUrl(config.UrlOverride.Reveal(), config.StageName) @@ -464,3 +481,161 @@ func (s *Service) GetCollectiblesService() *collectibles.Service { func (s *Service) GetCollectiblesManager() *collectibles.Manager { return s.collectiblesManager } + +// LocalBackup Code +func (s *Service) prepareSyncAccountMessage(acc *accounts.Account) *protobuf.SyncAccount { + return &protobuf.SyncAccount{ + Clock: acc.Clock, + Address: acc.Address.Bytes(), + KeyUid: acc.KeyUID, + PublicKey: acc.PublicKey, + Path: acc.Path, + Name: acc.Name, + ColorId: string(acc.ColorID), + Emoji: acc.Emoji, + Wallet: acc.Wallet, + Chat: acc.Chat, + Hidden: acc.Hidden, + Removed: acc.Removed, + Operable: acc.Operable.String(), + Position: acc.Position, + ProdPreferredChainIDs: acc.ProdPreferredChainIDs, + TestPreferredChainIDs: acc.TestPreferredChainIDs, + } +} + +func (s *Service) backupWatchOnlyAccounts() ([]*protobuf.Backup, error) { + accounts, err := s.accountsDB.GetAllWatchOnlyAccounts() + if err != nil { + return nil, err + } + + var backupMessages []*protobuf.Backup + for _, acc := range accounts { + + backupMessage := &protobuf.Backup{} + backupMessage.WatchOnlyAccount = s.prepareSyncAccountMessage(acc) + + backupMessages = append(backupMessages, backupMessage) + } + + return backupMessages, nil +} + +func (s *Service) ExportBackup() ([]byte, error) { + backup := &protobuf.WalletLocalBackup{} + + woAccountsToBackup, err := s.backupWatchOnlyAccounts() + if err != nil { + return nil, err + } + for _, d := range woAccountsToBackup { + backup.WatchOnlyAccounts = append(backup.WatchOnlyAccounts, d.WatchOnlyAccount) + } + + return proto.Marshal(backup) +} + +func mapSyncAccountToAccount(message *protobuf.SyncAccount, accountOperability accounts.AccountOperable, accType accounts.AccountType) *accounts.Account { + return &accounts.Account{ + Address: types.BytesToAddress(message.Address), + KeyUID: message.KeyUid, + PublicKey: types.HexBytes(message.PublicKey), + Type: accType, + Path: message.Path, + Name: message.Name, + ColorID: multiaccountscommon.CustomizationColor(message.ColorId), + Emoji: message.Emoji, + Wallet: message.Wallet, + Chat: message.Chat, + Hidden: message.Hidden, + Clock: message.Clock, + Operable: accountOperability, + Removed: message.Removed, + Position: message.Position, + ProdPreferredChainIDs: message.ProdPreferredChainIDs, + TestPreferredChainIDs: message.TestPreferredChainIDs, + } +} + +// TODO this is a duplicate of the code in messenger_handler. Should it be moved to a common place? +func (s *Service) handleSyncWatchOnlyAccount(message *protobuf.SyncAccount) (*accounts.Account, error) { + if message.KeyUid != "" { + return nil, ErrNotWatchOnlyAccount + } + + accountOperability := accounts.AccountFullyOperable + + accAddress := types.BytesToAddress(message.Address) + dbAccount, err := s.accountsDB.GetAccountByAddress(accAddress) + if err != nil && err != accounts.ErrDbAccountNotFound { + return nil, err + } + + if dbAccount != nil { + if message.Clock <= dbAccount.Clock { + return nil, ErrTryingToStoreOldWalletAccount + } + + if message.Removed { + err = s.accountsDB.RemoveAccount(accAddress, message.Clock) + if err != nil { + return nil, err + } + dbAccount.Removed = true + return dbAccount, nil + } + } + + acc := mapSyncAccountToAccount(message, accountOperability, accounts.AccountTypeWatch) + + err = s.accountsDB.SaveOrUpdateAccounts([]*accounts.Account{acc}, false) + if err != nil { + return nil, err + } + + return acc, nil +} + +func (s *Service) handleWatchOnlyAccount(message *protobuf.SyncAccount) error { + if message == nil { + return nil + } + + acc, err := s.handleSyncWatchOnlyAccount(message) + if err != nil { + return err + } + response := wakusync.WakuBackedUpDataResponse{ + WatchOnlyAccount: acc, + } + encodedmessage, err := json.Marshal(response) + if err != nil { + return err + } + event := walletevent.Event{ + Type: EventWatchOnlyAccountRetrieved, + Message: string(encodedmessage), + } + s.feed.Send(event) + + return nil +} + +func (s *Service) ImportBackup(data []byte) error { + var backup protobuf.WalletLocalBackup + err := proto.Unmarshal(data, &backup) + if err != nil { + return err + } + var errs []error + + for _, watchOnlyAccount := range backup.WatchOnlyAccounts { + err = s.handleWatchOnlyAccount(watchOnlyAccount) + if err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} diff --git a/tests-functional/README.MD b/tests-functional/README.MD index 126e3f3fd44..fe6e3b686a1 100644 --- a/tests-functional/README.MD +++ b/tests-functional/README.MD @@ -35,7 +35,7 @@ Functional tests for status-go ### Run tests - In `./tests-functional` run `docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --remove-orphans`, as result: * a container with [status-backend](https://github.com/status-im/status-go/pull/5847) will be created with endpoint exposed on `0.0.0.0:3333` - * status-go will use [anvil](https://book.getfoundry.sh/reference/anvil/) as RPCURL with ChainID 31337 + * status-go will use [anvil](https://book.getfoundry.sh/reference/anvil/) as RPCURL with ChainID 31337 * Status-im contracts will be deployed to the network * In `./tests-functional/tests` directory run `pytest -m rpc` @@ -86,7 +86,40 @@ sudo ln -s $HOME/.docker/run/docker.sock /var/run/docker.sock ## Import issues -If you see some import issues, make sure that you have all requirements installed from `requirements.txt` +If you see some import issues, make sure that you have all requirements installed from `requirements.txt` ### For PyCharm users Make sure that you made `test-functional` source folder (Right click > Mark directory as > Source folder) + + +## Reliability tests + +This framework also hosts a suite of reliability tests. See more info [here](https://github.com/status-im/status-go/blob/develop/tests-functional/tests/reliability/README.MD) + +## RuntimeError: Docker image 'statusgo-xyz:latest' not found + +First check with `docker images` that the image was created. If not, re-run the build command from [How to Run](#how-to-run) + +Second, if the image is present in the list, check if you Python env is not connected to the wrong Docker daemon. + +Run this in the shell: +```shell +echo $DOCKER_HOST +docker context ls +``` + +Run this in the Python env (`python`): +```py +import docker +print(docker.from_env().api.base_url) +``` + +You might see that Python is using something like: +- unix:///run/docker.sock +- vs your CLI using unix:///var/run/docker.sock + +If that is so, force Python to use the same Docker socket (from `docker context ls`): +```shell +export DOCKER_HOST=unix:///var/run/docker.sock +``` +Add this before launching your test runner diff --git a/tests-functional/clients/status_backend.py b/tests-functional/clients/status_backend.py index b99dc13f8a8..05b3dee761f 100644 --- a/tests-functional/clients/status_backend.py +++ b/tests-functional/clients/status_backend.py @@ -66,6 +66,13 @@ def __init__(self, await_signals=[], privileged=False, ipv6=USE_IPV6): self.ws_url = f"{url}".replace("http", "ws") self.rpc_url = f"{url}/statusgo/CallRPC" self.public_key = "" + self.mnemonic = "" + self.key_uid = "" + self.password = "" + self.display_name = "" + self.device_id = str(uuid.uuid4()) # In reality this is taken from the device, don't confuse with Status installation_id + self.device_platform = PushNotificationRegistrationTokenType.UNKNOWN + self.node_login_event = {} RpcClient.__init__(self, self.rpc_url) SignalClient.__init__(self, self.ws_url, await_signals) @@ -252,27 +259,30 @@ def _set_token_overrides(self, network, token_overrides): return network def extract_data(self, path: str): - if not self.container: - if os.path.exists(path): - return path - return None + if self.container: + return self.container.extract_data(path) - try: - stream, _ = self.container.get_archive(path) - except docker.errors.NotFound: + if not os.path.exists(path): return None - temp_dir = tempfile.mkdtemp() - tar_bytes = io.BytesIO(b"".join(stream)) + return path - with tarfile.open(fileobj=tar_bytes) as tar: - tar.extractall(path=temp_dir) - # If the tar contains a single file, return the path to that file - # Otherwise it's a directory, just return temp_dir. - if len(tar.getmembers()) == 1: - return os.path.join(temp_dir, tar.getmembers()[0].name) + def import_data(self, src_path: str, dest_path: str): + """ + Import a file from the host (src_path) into the container at dest_path. + If not running in a container, just copy the file locally. + """ + if self.container: + self.container.import_data(src_path, dest_path) + return - return temp_dir + # Not running in a container, just copy the file locally + if not os.path.exists(src_path): + raise FileNotFoundError(f"Source path '{src_path}' does not exist.") + + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + with open(src_path, "rb") as src, open(dest_path, "wb") as dst: + dst.write(src.read()) def _set_display_name(self, **kwargs): self.display_name = kwargs.get( @@ -326,6 +336,18 @@ def logout(self): method = "Logout" return self.api_valid_request(method, {}) + def wait_for_login(self): + signal = self.wait_for_signal(SignalType.NODE_LOGIN.value) + if "error" in signal["event"]: + error_details = signal["event"]["error"] + assert not error_details, f"Unexpected error during login: {error_details}" + self.node_login_event = signal + logging.debug(f"Node login event: {self.node_login_event}") + self.public_key = self.node_login_event.get("event", {}).get("settings", {}).get("public-key") + self.mnemonic = self.node_login_event.get("event", {}).get("settings", {}).get("mnemonic") + self.key_uid = self.node_login_event.get("event", {}).get("account", {}).get("key-uid") + return signal + def container_pause(self): if not self.container: raise RuntimeError("Container is not initialized.") diff --git a/tests-functional/resources/images/test-image-200x200.jpg b/tests-functional/resources/images/test-image-200x200.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c83855aec0ae10b90ad9520bf6926229d00be0b8 GIT binary patch literal 4925 zcmb7HcOcaN|G#s#!<{|r&fc7zgtK>NRze(SDJdOV-6=lkT-$sAx>?~bP%0AOS!0iXo{0Mq~$AP4{? zbEg*%km^rsOXhk1<{inr*xxb~WS*7cKY7oGlz-&O2LF&D`F?8WpOyvycuWJJB)^f9 zN5}ya|KfpU3lEZL0377^ugMo6_iq`5oE$LsFY6~K002eqU)FocBB1}_PgRgsIz~q1 z7Uk&@7~~5x(p7^g$|FuD0XhI$YHAv4Dq0#E8ag^!dN3O!n1KPz&BDse#?N#13_lMa zA52I_6eb`g$j2w9A}%F|P*hYrD|+sNs=T_4f};GX5g;8M9hd>k#mLAd59fo+|G(*^ z69APlx~>@<4!OIvv5^M4k*vDr#~c=V{0P_K>HN0(3F~0E2)4 z3J3^7ex5GcUA9q@CRqeAiFe`sP0Y1>T_6-XM)daiQ2PdzBKxF?$E`#8Q>sni;aH$R zpI+eum_UA{@p8$qiyw}$pWki4xbF#f6eOOO5!|H>HAeMG@-|7?&{5YnXDji!dp}ZW z;48-Pi3U1?Mu)dAJ*UaMd-ti@?zVct%UWhPM3wCIF{fM23-6yBy}O4`u1?KY$hL^L zk5NHYz{VD`rr76W=~;dBaGd%g+pgY^ts;kfhJN5(a%05|(VUVC?p{j5){$k-JfCOp`mRItLuBB{iUIT?+Q2VpE`sBqeQAy9f-qD z;QXs}Qm|nS8plE2xiIV_p7ny5{pPvz8J)j$5_jt!Z@9Au)y40PbpEDSe@`mgX1=Ur zdRQ^IOk*+F;yYkHD#6DjR%nT1N*?TqtP!U~`HzVtS9mylDX3U@EypKUN7{h}{<34Vd#M1=i&S^o>9DbG&X# z>M%K{h>(2LIVih04d39d5VC)79R+u0I{$_ z*x)b$IRqurkG$Bl?3~|5zETT3c6C7nNcz3Ehoob@2QXTP>h}6ulM3F zxQalqQx{l7M8r@+wBcL@Y%aUhB0@$Hk?e>c%WILeg3*ZO<0B;V4w5poW0ewOu<|z1 z#>i#@I&Of$|E74Qql|LR=`q<<4HB{mW!g0*92WK^xw`qdxqaeai%;d&EP;`YY(6P# z4Rkxawe*7zNeMy5ke!{j=vTLH*Y4~$DFlByPlMslG)ZtCFI-36l1}}cU#2qo?{QV- zs`9XJ?j*7iE8^zr<{B3>e_^=tK{_L6@Q6=o8?y~&YVPC)CAvY;h| z5@poY6>|IhWn*~N=a(jJKd6XRrWIm|7tg>ys#{qshztK%pGQW}d@&1YLe{XWd5L$( z(pdB64q+vI@TMP)Vs-sfc%j)M#Q3;zqsH;Zh?&Cahd0LsUF!s_*o}C(C(pAz@jy2x zHVl2c&1+Gcj21CSl(IRCM-<|8;E9$OgdZbgh0kIU`a;N6(VeY6`a&OzoD4Y0MBZ)8 zSDBl|a8h1gx1d8qeL)Z{5mX}-CU`T?#^j4xbt_1kwGY4}>6~@<<98TrQDSKsyhlHHWfr zM*BrX){gv*QEE+!-{^# z$K7D}?pE^KLV9|Z{|K@Eh|s3R=#omx(~?jr!gSh;GJnpnYeLCm*=czKldm#=E<|}a6w9ss@v+ffr4rG41SQRK@H@~yK{lqmPfOl=!X(x+Q zJnZ0{E)4r9i@?cOuSj4kRk|)=oHX__X{U14Q>VXJPm_h@lQBt1KAkKv2!OmnfhbQS z`QIppLl8&{XaB?qSYFL{xsk1oxql*A6aK*?s%Ru!aAYkiA0uvTp*dSLva+K+meOh& zLol`^n3R8@)ssolWBuTLv*=K)2Og?eB(D3`qBf;S+@u`R9=VcXQZz<>FouUh5>{|y zkT~2Jo`Vda|k42{PI=Lo)jOWkM%@j5Cl#tJg)=Yf%17g>S(yBM?Pf%WnU z&j=ptU}2U}aB)xE0GGbN1XW}D1904h`irYrvD$tXbiK-FuB&u;r7!dfN&4=l8S+8{ zeHR7H>798KyIJC)&z`S4!S=R#ypPI76OS@-3T%D9{rw+#!u$g6H41jpHEZAd1( zIiKTA`cT@I^7ToXCS8FP^Op1PFuQ`i8NLOJ8eM0I@TOnXCxCUyuR9kjXtg0YkwY7W z+W6f{>G>|{5eqEk=;+nKf&+BIKf2;X)SJP$(y1&Uho%2fbZrJ)NLPh#0&%M{|mPe?&X=F(3BD=DzG=D!u zhybC*!T2!s^Sk6ehL$siZ3*t-o8htHv4QPqPD4&T4bvA|%MrIr+&L;sZYM9c9gk8a z9OsWyK05)NBVhY}S*riyU;3t#R)zkW=O@<+yl&X;YFcO9FgM&Gwk0!tFW#T2+DzDF zC1(Nl9rjVc?RNx0*t7K-C_AkCVua$9?yK!{{?64gPZ{hsU?0!={X`3gQN9ZaNnrO4 zi66WnWh0-o%^kkt5T3-#99%xWRHe7cz*oHDF=ILz93D}Cvk&iLxvc?AG?J=Ux#TWK zw7OSWWK`&mLrqSz2fJe5jvsLPAtR>a-=xyAMCITtC$}FR3@yCm1Q|r!e1m<~sJ%px z`&}#9nIMG%zv@tW%O(=r@{md6GF~I@VepIH8@;}b?bO}(9@Xu!sN_i9E2gS9mtkvT z+Dh4%5dQT5T%GaiCl8n263eu})%F&%gPFuHJC_hmH=~7bRWQaai;tXT6HjYfuw2``^jPV&oX4XqsPK1 z&lg?~Z)##x&UE=g$A{i<9!I!M3K8@zuUawYSSZc?JijQ~!1&}W*h*tBh>)KEOX-kw z8a_PY9PWAk1zu&Vh~AqL>~J6wB7Y8B!<|a5+(=VA;HW0X_Z{DcGrhqHO;*)(gg|S5 z!HbgGsl^lE#VfOIhHcK*3p_JL}g zpTv||F{uG#iWACcJn%QHO9IMryS1e!u=H=V_HLH-OSN7dU=xiBh?c1H zEU21#Hnro1DZbX#V5@h=S+(vpSo6L5_+m_7 za}={efuDW)i=1b8q2w?5H^uzP%RnduCI{F0lUh!90thVHZ0@{ybsP>(mm4Q~<-5A+ zbEpY*7dlofE^cRM$51(10v1UhOQz@p7eO+?Y6Xbex}t1|z(0G&U$y8zX^iaS)B{`& zf&Am*bkBgCF&8n*(##f{=}9Ti$`)jxio$#yZ0G8$(v69Vb%g(Zh>H~w>NKa)_9%eM zL*f#m;&2OF4musnS(aM|S%bJ9b@bc~vNKT#xrU`6*FWUa|8#~=e-oha)2s>ck4Vhx zs2Q1qZGHbYYeLASzY2RbdT>BN<|a0}@(uxIAz9H5Hh5L;jHQ(4nTfYgC-; zaIx0@1tG?@mB1m_IVg~*W8prEmbIeQH+`w4t^OE)0*H8fEiaH`p6GIksw6sNJdXL;`#_*aD{Jj)Owr^IfwHLw&j!C#_$VH#CM`FpcNZG=g5HpeYaoog! zej>9;37vaDA8ubSjX-K5z5if~&f8(E-4v*H&?&PbJzKhf43Z`tr0mMd205l4raf}H zV4@oBg9q8Am{f1;L}Odqb$PGh&OcU+SK{qF0gUiZ;8kF4mXB({M*;zFR4;#*zrTpL zQBv&I^T;;Rt1^gb@SGbcPMBBIr6NtUEH_JieNSUmf>%NE$X~9akPJ?%Ceg-f#5GaM z=UMPxni{6I{CII7=&g%FCf+!o5l!t@^pGoxTfm8OY?iN^2~CLc7l5UCU7J_KW7Lh6 zOsCI9`d}ZVa@XJgfTnZA`gJOka>wAEh;!iL(sh?ncdXwzlMEk;>yY4BZ^*diKpB6c zPV^V`V1jatG&fdSfhxvIWZv5cAuU}KAD_xn>Y9h!@X|QIb*CT1G+ODS^hAraRnQ3% zZ_}&dt__UM^o0{Os5T0@{`;o(WosUoU!*(Iu-@<(N2A6u+R9s_+7qio0*S+9_>uY4E%JA~X0N z=I3uM`(t58%=wghiU?o36+#+e&5^nEdQs8re7a=Xjx_0Ib*_}{RC{XFT2U}o5%%Fr z`I)WsuwJv2X<)*c;k!5QrE(u|Sg4M@XL)Gm(j&ar(DZ=88U`%8XZ&P#wLcnT zw_JaAU+e@>@<4N(;RI0YaCpTt8Xu#%U3UVIW@0X&l)hBURvr?L({bYg{?@h2kq!6f z?wFq;@$-em>jLX~g2YRM^DCV{N4!Q1>~jLdBEQfM-$&<=ILLnrrGbRVrKlt4SB1^I zuf4v0qt1Sces6up3D+HlnP)=;8;o7udOQ<*s5L;qVFIBO!dV9&9vA}3tO6GLCWw%- z5rthb<0NN;pyb#6BjM40a1H+-sM^W3+NjJ4^rGYu_ch54KJ98>tLa^uT#FB@B>)$owIk-qB%(X$fU&Da`e(XbClYZBT}PP181SE9}mxRJPZ-u3YDX& zgV3>sD#A41+hfsENTKZRc>-BH=;U=mA#u71a#VKd9OHy-}| zSeD{=IKMfw5goDZQW)wVbW}ZJdjhCjQr4McEuof|yH|M+thINioN79_Wb?`wiiz_+ z&eJyIweHeQ1Ny|ZHLh|tI%eumArFYCP<|wh2@JF#l4n_NPw$4hpPlx*bC!0CE4^XE zhTh92kI1nnsOcyBOJQHE^33B3CN5RlM!WA6WubUjvCzv#Vo!I60akO39(@K%Jvqq= z@B0Sra!gYG`lTrx!WNq7eUNtP`4_2j1CA4b1>I$w_0KF_>U6P=uA|&3p8^;Cd?O*2^t2{JgW6n0{NUj&wXnX~a zX+o8?8*PRHn|1GSADF>Rn?x6q6gv2n{7loorQ6!x>qlKf+2$t|j|4n08WxyfAO21| z(qh|=7^vAupAF}Z>^?(R>hHE6s9F=N{Bn|}#vXKypb#TFtWpx=In z-&s{Kuj-~`D=$%%w;DrOVZJr@!F%2{&w#DF&ZSF!lxEs_mIp?hOgB;AXDLcm-=fA( z#XpO?-Q3t;dBY0m^6ZkA?UTb)ZeX^R8y$c#ZOl#Bwdt+b722X1`OY6(_`Kvy7^`6J gkf2cr)RuRXjKAn_G{jeS@mG{4Ts|e?15d{P56YEGga7~l literal 0 HcmV?d00001 diff --git a/tests-functional/resources/test_data/profile_showcase_utils.py b/tests-functional/resources/test_data/profile_showcase_utils.py new file mode 100644 index 00000000000..a7b02a65495 --- /dev/null +++ b/tests-functional/resources/test_data/profile_showcase_utils.py @@ -0,0 +1,93 @@ +def dummy_profile_showcase_preferences(with_collectibles: bool): + preferences = { + "communities": [ + { + "communityId": "0x254254546768764565565", + "showcaseVisibility": 3, + "order": 0, + }, + { + "communityId": "0x865241434343432412343", + "showcaseVisibility": 2, + "order": 0, + }, + ], + "accounts": [ + { + "address": "0x0000000000000000000000000033433445133423", + "showcaseVisibility": 3, + "order": 0, + }, + { + "address": "0x0000000000000000000000000032433445133424", + "showcaseVisibility": 2, + "order": 1, + }, + ], + "verifiedTokens": [ + { + "symbol": "ETH", + "showcaseVisibility": 3, + "order": 1, + }, + { + "symbol": "DAI", + "showcaseVisibility": 1, + "order": 2, + }, + { + "symbol": "SNT", + "showcaseVisibility": 0, + "order": 3, + }, + ], + "unverifiedTokens": [ + { + "contractAddress": "0x454525452023452", + "chainId": 11155111, + "showcaseVisibility": 3, + "order": 0, + }, + { + "contractAddress": "0x12312323323233", + "chainId": 1, + "showcaseVisibility": 2, + "order": 1, + }, + ], + "socialLinks": [ + { + "text": "TwitterID", + "url": "https://twitter.com/ethstatus", + "showcaseVisibility": 3, + "order": 1, + }, + { + "text": "TwitterID", + "url": "https://twitter.com/StatusIMBlog", + "showcaseVisibility": 1, + "order": 2, + }, + { + "text": "GithubID", + "url": "https://github.com/status-im", + "showcaseVisibility": 2, + "order": 3, + }, + ], + } + + if with_collectibles: + preferences["collectibles"] = [ + { + "contractAddress": "0x12378534257568678487683576", + "chainId": 1, + "tokenId": "12321389592999903", + "showcaseVisibility": 3, + "order": 0, + } + ] + else: + preferences["collectibles"] = [] + + return preferences diff --git a/tests-functional/resources/test_data/test_old_user_data.bkp b/tests-functional/resources/test_data/test_old_user_data.bkp new file mode 100644 index 0000000000000000000000000000000000000000..adef914bed5f7bc8017e66c71ab2209c0951e26f GIT binary patch literal 2622 zcmV-E3c>XPg`=`CE`Kt0%Dbggysnq8{l1|3)#;-%MmO6YZ7q-$8HzV)&x!Vm1D^lv ziy^9Q`P$W7FYA0$Jo4)$ z$;OI~2^irQ#32pr{xpib9+s**96_Cv_h*j38nolw_>u%$!i6BUAo~C7yVF3f$e4L% z)tvy0kjf7CDJc_EQdWwyhaY+9#YLt*DMmWRY_(q2LqyY}k7C?9J-D-mM|}8#i`@PV z+HGs!PEB>$xn$Tb(V2}N2ot;DsSN61*ZDf)FD`-C3Q#zH{tOFuuC=B`ubDWvY#w)1 z?|Nqqi)#Hy*@lC?Hv9?)f0y8tpHW(e>X+jlr2)v@Dz~1NeaWCNWacl>&fIb7m!_YY zs#B!S*t?5cT(uW~fp|qeCSYchj3E`l{k4$|bMYL5M@K?!y?4uR{Gh+@nJ2g+@I@n& zT!wGMZTM6knGci42yu{LgAgb5o(Z4F+&!1xe z*qR|a#`}0sf}COYUuok(-%G3!|6`|9e1)h2Qq*T$acT*iT={E=F2KZ!%`$_f6zeAq zX{n(+g_|R(#o?*^kkp&qLEu51{Mj`0gq{*@S_l20t;I}qy=e!HjQh!LiF~vJ>c6iu z5X&$PlEV8?_r$PRa5d+JH?zfWNWs`<33H%-FDwZ7YqJHSXh1hg_*G)wpm$|NJ^0|E zbN1J>`|T{t9KKQSODTa*f9|4zuX$3hH_5Qikx@Mff0Y9#I_Lz)691SD2wp}W7u)BX zONh*dkaq{wRUF8EOp{_!{ZqO{sD*Z8Q)3(ky4z?yeY8o& zl?9V*oil zz^a5fJb)$!s5${zCok>hP_hOX3$t379AEfq=Eh!n-qSdYo9V8X1nN-Fu%|j}KD#j< zK?W8AUstQ{d}}&pZiNe$7t4nYg2|A37u$*ilSd(AXy94#O!%gRR>1KoAk_8qb)Dwm z1?Mw0rf)~iUdp~KvD|3@>qq|}SNeJSw+x8Flq07u;%OcacYyl5BN+X1iVrtPPAu_k zB5n2}@m!kTP5=xVvep48#KI|#iCu?ix@iIaOg0u!m`d_^x+&30@#Ij$hZmAU*z4gN z;C)k`yEB0Ww4G$g^8|Ol6U_ROW+(1HKALbP#bJXTW{q@2zbh8344QrN7Eb(|Ieqc~ z^hfR?OYYtG@C~Q8qfl!M^o)CK@MWXngvEJ@fGXBzo6d6WLR06Az-?WfcTVi9+(Q%e-fR#c% z4R)dspY8A_d5F#MV%qc14jGf${LD?o0!>IS(;pxYMo*@UV};0+YVB zPt|Aayfd70nO70aQBTdjJ}yiQU}Dvk119v$`jr5M3vs$*m=iPdu)smIjoTJ!Bnb%S zynffFRi6#c^Lr2Jc9CdS2D zO3|is^u7#n6?A>j;x2`7n0N0)8rI=b=i@=Ct($8gYXKp20UYCzW)0K6g1*M!)yk4Tyq|A+i6-3M1Ah-Oa4d(@&!Nse6hyNyGj|w9 zm=Z|&1^L`Bp}frkX7mrax$l`zWIDg7 zs=|gitJ#<--$gP(^X;s!DJN_~3&Lmm9AiaCUQMdO5-+Sx8%8_Q&GRGC<|YYqocO@P zJwJgkZZCt{bO*ErO4ZuWSBpGv_9?x9|_WIah*)VOCw0 zFd-vJVyc4Cy2Etc6RD%ub)Cu`*Qf?;rQ)$TL1hm*wv>6{AavKpHZkU2 zIE!MCwL~s|=yPEr*3XpWc4Y1x6NB3d(A^emivBXQ) zgz(_15@mEMEOdasQ1Bc9sCiJG_4k7XME-R($(TaCLMR!Iux!pdd#lq#E3)bXDV>Gk zKt&UbMmG!wVCOPCi^?kw0t-X*JDow?pzqa|7({yjeYIca|%u14_ddvv`$ z-fe5>mT?+NpQ0I&76VWE3%g(rQI0G<@6^=&S1!(UP){310W`$rnhyoR&F^v+*tz$L z1Tu@$#3ZGdqks%pp$RLj2wEtEl3X-$LF`yD>8PKe`4Yz2WcbKIA;NDSUeeQvy*YuX zXMBvm%SMO2$$x&_?z^;7uKDz?pROeXqHyMu20_5Q*W!$j z*F&gGa>;Fu-~!Jjlqvq|W6u=poK5=0YE_5Bf?RDHCQk}_-X08*r#Tx&vWrDd5HU 0, "Clock is 0 in the profile showcase preferences" + profile_showcase_preferences["clock"] = result.get("clock") # override clock for simpler comparison + assert result == profile_showcase_preferences, "Profile showcase preferences were not set correctly" + + # Validate watchonly account + response = backend_client.rpc_valid_request("accounts_getAccounts", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Accounts were not restored" + assert len(result) == 3, "Watchonly account was not restored" + assert result[2].get("name") == "test-watchonly-account", "Watchonly account name was not set correctly" + assert result[2].get("emoji") == "🕵️", "Watchonly account emoji was not set correctly" + assert result[2].get("type") == "watch", "Watchonly account type was not set correctly" + + # Validate contacts + response = backend_client.rpc_valid_request("wakuext_contacts", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Contacts were not restored" + assert len(result) == 2, "Contacts were not restored" + + # Validate communities + response = backend_client.rpc_valid_request("wakuext_communities", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Communities were not restored" + assert len(result) == 2, "Communities were not restored correctly" + assert result[0].get("name") == "Test Community", "Community name was not restored correctly" + assert result[0].get("description") == "This is a test community", "Community description was not restored correctly" + assert result[1].get("name") == "Test Community 2", "Second community name was not restored correctly" + assert result[1].get("description") == "This is another test community", "Second community description was not restored correctly" + assert not result[1].get("joined"), "Second community was not left correctly" + + # Validate chats + response = backend_client.rpc_valid_request("wakuext_chats", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Chats were not restored" + group_chat_recovered = False + one_on_one_chat_recovered = False + for chat in result: + if chat.get("id") == group_chat_id: + assert chat.get("name") == "test-group-chat", "Group chat name was not restored correctly" + group_chat_recovered = True + elif chat.get("id") == test_one_to_one_chat_id: + one_on_one_chat_recovered = True + + assert group_chat_recovered, "Group chat was not restored correctly" + assert one_on_one_chat_recovered, "One-to-one chat was not restored correctly" + + # Perform the backup + response = backend_client.api_valid_request("PerformLocalBackup", "") + jsonResponse = response.json() + backup_path = jsonResponse.get("filePath", "") + + # Extract the backup file from the container + backup_file = backend_client.extract_data(backup_path) + assert backup_file is not None + + # Create a new installation + backend_client2 = StatusBackend(await_signals) + assert backend_client2 is not None + + # Init and login + backend_client2.init_status_backend() + backend_client2.restore_account_and_login( + user=Account( + passphrase=user_mnemonic, + password=backend_client.password, + # The private key and address are not needed for mnemonic recovery + private_key="", + address="", + ) + ) + backend_client2.wait_for_login() + backend_client2.wakuext_service.start_messenger() + + # Import the backup file into the new container + backend_client2.import_data(backup_file, "/usr/status-user/backups") + + # Import the backup file + response = backend_client2.api_valid_request("LoadLocalBackup", {"filePath": backup_path}) + + # Validate profile settings + response = backend_client2.rpc_valid_request("multiaccounts_getIdentityImages", [backend_client2.key_uid]) + jsonResponse = response.json() + result = jsonResponse.get("result", {}) + assert result is not None, "Identity images were not restored" + assert len(result) == 2, "Identity image was not restored" + + # Validate settings + response = backend_client2.rpc_valid_request("settings_getSettings", []) + jsonResponse = response.json() + result = response.json().get("result", {}) + assert result is not None, "Settings were not restored" + # FIXME display name doesn't work because we set the display name when creating the account + # and the clock is somehow too low, so the display name is not updated + # assert result.get("display-name") == "myname", "Display name setting was not restored" + assert result.get("bio") == "new bio", "Bio setting was not restored" + assert result.get("preferred-name") == "new-prefered-name", "Preferred name setting was not restored" + assert result.get("url-unfurling-mode") == 2, "URL unfurling mode setting was not restored" + assert result.get("messages-from-contacts-only") is True, "Messages from contacts only setting was not restored" + assert result.get("currency") == "eth", "Currency setting was not restored" + assert result.get("mnemonic-removed?") is True, "Mnemonic setting was not restored" + + # Validate profile showcase preferences + response = backend_client2.rpc_valid_request("wakuext_getProfileShowcasePreferences", []) + jsonResponse = response.json() + result = response.json().get("result", {}) + assert result is not None, "Profile showcase preferences were not restored" + assert result.get("clock") > 0, "Clock is 0 in the restored profile showcase preferences" + profile_showcase_preferences["clock"] = result.get("clock") # override clock for simpler comparison + assert result == profile_showcase_preferences, "Profile showcase preferences were not restored correctly" + + # Validate watchonly account + response = backend_client2.rpc_valid_request("accounts_getAccounts", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Accounts were not restored" + assert len(result) == 3, "Watchonly account was not restored" + assert result[2].get("name") == "test-watchonly-account", "Watch only account name was not restored correctly" + assert result[2].get("emoji") == "🕵️", "Watch only account emoji was not restored correctly" + assert result[2].get("type") == "watch", "Watch only account type was not restored correctly" + + # Validate contacts + response = backend_client2.rpc_valid_request("wakuext_contacts", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Contacts were not restored" + assert len(result) == 2, "Contacts were not restored correctly" + + # Validate communities + response = backend_client2.rpc_valid_request("wakuext_joinedCommunities", []) + jsonResponse = response.json() + result = jsonResponse.get("result", {}) + assert result is not None, "Communities were not restored" + assert len(result) == 1, "Communities were not restored correctly" + assert result[0].get("name") == "Test Community", "Community name was not restored correctly" + assert result[0].get("description") == "This is a test community", "Community description was not restored correctly" + + # Validate chats + response = backend_client2.rpc_valid_request("wakuext_chats", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Chats were not restored" + group_chat_recovered = False + one_on_one_chat_recovered = False + for chat in result: + if chat.get("id") == group_chat_id: + assert chat.get("name") == "test-group-chat", "Group chat name was not restored correctly" + group_chat_recovered = True + elif chat.get("id") == test_one_to_one_chat_id: + one_on_one_chat_recovered = True + assert group_chat_recovered, "Group chat was not restored correctly" + assert one_on_one_chat_recovered, "One-to-one chat was not restored correctly" + + def test_local_backup_backwards_compatibility(self, tmp_path): + await_signals = [ + "mediaserver.started", + "node.started", + "node.ready", + "node.login", + ] + backend_client = StatusBackend(await_signals) + assert backend_client is not None + # Restore the account with a hardcoded mnemonic to get the same private key + backend_client.init_status_backend() + backend_client.restore_account_and_login( + user=Account( + # Hardcoded mnemonic for backwards compatibility test + # Do not change this mnemonic, the backup file was created with it + passphrase="flush okay stadium refuse shrug fit actual stairs puzzle brisk flat moral", + password=user_1.password, + # The private key and address are not needed for mnemonic recovery + private_key="", + address="", + ) + ) + backend_client.wait_for_login() + backend_client.wakuext_service.start_messenger() + + # Import the backup file into the container + backend_client.import_data( + os.path.abspath(os.path.join(os.path.dirname(__file__), "../resources/test_data/test_old_user_data.bkp")), "/usr/status-user/backups" + ) + + # Import the backup file + # This is the old backup file that does not contain the BackedUpProfile protobuf + # This means that the display name, identity image and profile showcase preferences will not be restored + response = backend_client.api_valid_request( + "LoadLocalBackup", {"filePath": os.path.join("/usr/status-user/backups", "test_old_user_data.bkp")} + ) + + # This "old" backup file does not contain the identity image, so we expect it to be empty + response = backend_client.rpc_valid_request("multiaccounts_getIdentityImages", [backend_client.key_uid]) + jsonResponse = response.json() + result = jsonResponse.get("result", {}) + assert result is None, "Identity images shouldn't be restored from the old backup file" + + # Profile showcase preferences were not set in the old backup file, so we expect it to be empty + response = backend_client.rpc_valid_request("wakuext_getProfileShowcasePreferences", []) + jsonResponse = response.json() + result = response.json().get("result", {}) + assert result is not None, "Profile showcase preferences were not restored" + assert result.get("clock") == 0, "Clock should be 0 as it was not set in the old backup file" + assert result.get("accounts") is None, "Accounts should not be set in the old backup file" + assert len(result.get("collectibles")) == 0, "Collectibles should not be set in the old backup file" + assert result.get("communities") is None, "Communities should not be set in the old backup file" + assert len(result.get("socialLinks")) == 0, "Social links should not be set in the old backup file" + assert len(result.get("unverifiedTokens")) == 0, "Unverified tokens should not be set in the old backup file" + assert len(result.get("verifiedTokens")) == 0, "Verified tokens should not be set in the old backup file" + + # The rest of the data should be restored correctly, so we can check the settings, watchonly account, contacts, communities and chats + # Validate settings + response = backend_client.rpc_valid_request("settings_getSettings", []) + jsonResponse = response.json() + result = response.json().get("result", {}) + assert result is not None, "Settings were not restored" + assert result.get("bio") == "new bio", "Bio setting was not restored" + assert result.get("preferred-name") == "new-prefered-name", "Preferred name setting was not restored" + assert result.get("url-unfurling-mode") == 2, "URL unfurling mode setting was not restored" + assert result.get("messages-from-contacts-only") is True, "Messages from contacts only setting was not restored" + assert result.get("currency") == "eth", "Currency setting was not restored" + assert result.get("mnemonic-removed?") is True, "Mnemonic setting was not restored" + + # Validate watchonly account + response = backend_client.rpc_valid_request("accounts_getAccounts", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Accounts were not restored" + assert len(result) == 3, "Watchonly account was not restored" + assert result[2].get("name") == "test-watchonly-account", "Watch only account name was not restored correctly" + assert result[2].get("emoji") == "🕵️", "Watch only account emoji was not restored correctly" + assert result[2].get("type") == "watch", "Watch only account type was not restored correctly" + + # Validate contacts + response = backend_client.rpc_valid_request("wakuext_contacts", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Contacts were not restored" + assert len(result) == 2, "Contacts were not restored correctly" + + # Validate communities + response = backend_client.rpc_valid_request("wakuext_joinedCommunities", []) + jsonResponse = response.json() + result = jsonResponse.get("result", {}) + assert result is not None, "Communities were not restored" + assert len(result) == 1, "Communities were not restored correctly" + assert result[0].get("name") == "Test Community", "Community name was not restored correctly" + assert result[0].get("description") == "This is a test community", "Community description was not restored correctly" + + # Validate chats + response = backend_client.rpc_valid_request("wakuext_chats", []) + jsonResponse = response.json() + result = jsonResponse.get("result", []) + assert result is not None, "Chats were not restored" + # Chat ids are hardcoded becuase they are in the backup file, so we can check them directly + test_group_chat_id = "1fb5f429-1e23-4156-b216-2799fa430742-0x045b1f2d9bfbbeee1f6995a2086de353399074073d213079d3e0dc2ddc1593e7c50e56aead4a42e25a76effa4208f842f9356f1a5c16aaf16c179ed020277a9901" # noqa: E501 + group_chat_recovered = False + one_on_one_chat_recovered = False + for chat in result: + if chat.get("id") == test_group_chat_id: + assert chat.get("name") == "test-group-chat", "Group chat name was not restored correctly" + group_chat_recovered = True + elif chat.get("id") == test_one_to_one_chat_id: + one_on_one_chat_recovered = True + assert group_chat_recovered, "Group chat was not restored correctly" + assert one_on_one_chat_recovered, "One-to-one chat was not restored correctly" From 3d445349f00efb3bad90e5c2ccc1c99933320cc9 Mon Sep 17 00:00:00 2001 From: fbarbu15 Date: Mon, 4 Aug 2025 14:39:27 +0300 Subject: [PATCH 02/15] test_: backup mnemonic and restore account (#6789) * test_: add restore account * test_: add restore account tests * test_: add restore account tests * test_: add concurrent test * test_: class rename * test_: fix assert * test_: review comments * test_: fix * test_: fix review comments --- tests-functional/resources/constants.py | 49 +++++ .../resources/test_data/__init__.py | 1 + .../resources/test_data/mnemonic_12.py | 61 ++++++ .../resources/test_data/mnemonic_15.py | 39 ++++ .../resources/test_data/mnemonic_24.py | 39 ++++ .../tests/test_backup_mnemonic_and_restore.py | 173 ++++++++++++++++++ 6 files changed, 362 insertions(+) create mode 100644 tests-functional/resources/test_data/__init__.py create mode 100644 tests-functional/resources/test_data/mnemonic_12.py create mode 100644 tests-functional/resources/test_data/mnemonic_15.py create mode 100644 tests-functional/resources/test_data/mnemonic_24.py create mode 100644 tests-functional/tests/test_backup_mnemonic_and_restore.py diff --git a/tests-functional/resources/constants.py b/tests-functional/resources/constants.py index 84709b7b52e..bc3c3709b53 100644 --- a/tests-functional/resources/constants.py +++ b/tests-functional/resources/constants.py @@ -1,5 +1,8 @@ +# Main constants file for tests from dataclasses import dataclass import os +from typing import Optional, List, Dict, Any +from resources.test_data import mnemonic_12, mnemonic_15, mnemonic_24 @dataclass @@ -8,6 +11,8 @@ class Account: private_key: str password: str passphrase: str + accounts: Optional[List[Dict[str, Any]]] = None # Optional list of accounts + profile_data: Optional[Dict[str, Any]] = None # Optional profile data user_1 = Account( @@ -22,6 +27,50 @@ class Account: password="Strong12345", passphrase="test test test test test test test test test test nest junk", ) + +user_mnemonic_12 = Account( + address="0xC43f4Ab94eC965a3EE9815C5Df07383057d261A8", + private_key="", + password="Strong12345", + passphrase="exhibit soldier miracle series edge atom daring alter absorb decide orphan addict", + accounts=mnemonic_12.accounts, + profile_data=mnemonic_12.profile_data, +) + +user_mnemonic_15 = Account( + address="0x685d7ec8e08769ca7020a6b65709887e38e68e6d", + private_key="", + password="Strong12345", + passphrase="category two chapter fame hunt horse huge rotate inner monkey affair champion mixed tail final", + accounts=mnemonic_15.accounts, + profile_data=mnemonic_15.profile_data, +) + +user_mnemonic_24 = Account( + address="0xf2d58ae5aa880f7c3f65d769296b1061c61e0955", + private_key="", + password="Strong12345", + passphrase=( + "border cabbage grape stage return enable bamboo main only voyage glad race patient stool drum sort " + "army abandon elegant grit cinnamon endless rail drink" + ), + accounts=mnemonic_24.accounts, + profile_data=mnemonic_24.profile_data, +) + +user_keycard_1 = { + "keyUID": "5a0dd657-165a-4810-b800-6005452be42f", + "address": "0x1234567890abcdef1234567890abcdef12345678", + "whisperPrivateKey": "example-whisper-private-key", + "whisperPublicKey": "example-whisper-public-key", + "whisperAddress": "example-whisper-address", + "walletPublicKey": "example-wallet-public-key", + "walletAddress": "0xabcdefabcdefabcdefabcdefabcdefabcdefabcd", + "walletRootAddress": "0xrootaddressrootaddressrootaddressrootaddr", + "eip1581Address": "0xeip1581address1234567890abcdef1234567890", + "encryptionPublicKey": "example-encryption-public-key", +} + PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) TESTS_DIR = os.path.join(PROJECT_ROOT, "tests-functional") SIGNALS_DIR = os.path.join(TESTS_DIR, "signals") diff --git a/tests-functional/resources/test_data/__init__.py b/tests-functional/resources/test_data/__init__.py new file mode 100644 index 00000000000..02b3d28c5aa --- /dev/null +++ b/tests-functional/resources/test_data/__init__.py @@ -0,0 +1 @@ +# Make test_data a proper Python package diff --git a/tests-functional/resources/test_data/mnemonic_12.py b/tests-functional/resources/test_data/mnemonic_12.py new file mode 100644 index 00000000000..aaba1ae739e --- /dev/null +++ b/tests-functional/resources/test_data/mnemonic_12.py @@ -0,0 +1,61 @@ +# Account data for mnemonic with 12 words + +accounts = [ + { + "address": "0xb01a7dbaaacc92581558a4d178289be7471ba0f4", + "public-key": ( + "0x047fd6e4384b764cb7782bf747ad3fb51e8c54c2b3c2be7029c72d85d16b07f4be6a8703cf0a1d97be4c8712ad" "52cd6a519efdc723faae4935bbaba5c320dde02b" + ), + "path": "m/43'/60'/1581'/0'/0", + "prodPreferredChainIds": "1:10:42161:8453", + "operable": "fully", + "position": -1, + }, + { + "address": "0xc43f4ab94ec965a3ee9815c5df07383057d261a8", + "public-key": ( + "0x041d8f6ebfa662c506d3d11cd407373f8ff2eb4c9ddc61a834864150a908172efbaa37f0de7dbeeea51f02272" "74e6ccd78fa5a07de55716094dbba475ac9e9ab44" + ), + "path": "m/44'/60'/0'/0/0", + "name": "Account 1", + "colorId": "primary", + "hidden": False, + "prodPreferredChainIds": "1:10:42161:8453", + "position": 0, + }, +] + +profile_data = { + "address": "0x5f8e02f9f52709b29c82bd893d9af0f83273a6e4", + "currency": "usd", + "networks/current-network": "", + "dapps-address": "0xc43f4ab94ec965a3ee9815c5df07383057d261a8", + "eip1581-address": "0x803102f704324d4a4f2ddb1c47f6ab31339d0f70", + "key-uid": "0x3231d92c94548d14f097173765a50bebe28fbad8f2267c9e08cc4433a6f219a4", + "latest-derived-path": 0, + "link-preview-request-enabled": True, + "messages-from-contacts-only": False, + "mutual-contact-enabled?": False, + "name": "Anchored Open Wirehair", + "networks/networks": [], + "news-feed-enabled?": True, + "news-rss-enabled?": True, + "photo-path": "", + "preview-privacy?": False, + "public-key": ( + "0x047fd6e4384b764cb7782bf747ad3fb51e8c54c2b3c2be7029c72d85d16b07f4be6a8703cf0a1d97be4c8712ad52cd" "6a519efdc723faae4935bbaba5c320dde02b" + ), + "default-sync-period": 777600, + "appearance": 0, + "profile-pictures-show-to": 2, + "profile-pictures-visibility": 2, + "use-mailservers?": True, + "wallet-root-address": "0x1846a7930d0ab03e5a120ebdd46eff3fe0365824", + "send-status-updates?": True, + "backup-enabled?": True, + "show-community-asset-when-sending-tokens?": True, + "display-assets-below-balance-threshold": 100000000, + "url-unfurling-mode": 1, + "compressedKey": "zQ3shoF8xQNaT44MWQxztUXK6DK9UU63PRgJhn1Zd7oYWXJ5K", + "emojiHash": ["🔢", "🤞🏽", "🖍️", "🙇🏽‍♀️", "🙋🏼‍♀️", "🙌", "💎", "🎞️", "😊", "🦸🏼", "😤", "👵🏿", "🧑🏿‍🔧", "🤶🏿"], +} diff --git a/tests-functional/resources/test_data/mnemonic_15.py b/tests-functional/resources/test_data/mnemonic_15.py new file mode 100644 index 00000000000..fc18fcd62af --- /dev/null +++ b/tests-functional/resources/test_data/mnemonic_15.py @@ -0,0 +1,39 @@ +# Account data for mnemonic with 15 words + +accounts = [ + { + "address": "0x245b7438961de05444645898f9215e5cf0786891", + "public-key": ( + "0x04c898c7763afd577f10efdd9e5d607caafd6d708e6cad8cee1b6d822d6ab148eb4e76d1c8266c8b73c8ce1d76" "699e072ac5844a9a6934abb565044bf619336302" + ), + "path": "m/43'/60'/1581'/0'/0", + "prodPreferredChainIds": "1:10:42161:8453", + "operable": "fully", + "position": -1, + }, + { + "address": "0x685d7ec8e08769ca7020a6b65709887e38e68e6d", + "public-key": ( + "0x0480acddfad1e73c3e70e8e50f82eb1566e3df125736e9fe9042c4df5022c825afe6234021ad8bbb43e0ab0196" "878c2d9e3c8b5a8f266aca72b0e23d1f84464c72" + ), + "path": "m/44'/60'/0'/0/0", + "name": "Account 1", + "colorId": "primary", + "hidden": False, + "prodPreferredChainIds": "1:10:42161:8453", + "position": 0, + }, +] + +profile_data = { + "address": "0x3644a8cc3860606fdee3b95c8825e17933a91647", + "dapps-address": "0x685d7ec8e08769ca7020a6b65709887e38e68e6d", + "eip1581-address": "0xe98734898ff58ac33a1a9c28f732696ec3e6b580", + "key-uid": "0x944c1ce03f83dd1750acee591745d6ef14da90723af86f97b2df7d7282e8dd97", + "name": "Overcooked Lost Grayreefshark", + "public-key": ( + "0x04c898c7763afd577f10efdd9e5d607caafd6d708e6cad8cee1b6d822d6ab148eb4e76d1c8266c8b73c8ce1d76699e" "072ac5844a9a6934abb565044bf619336302" + ), + "wallet-root-address": "0xe89675c9be641ceeca9f250345dc58528c3de93b", + "emojiHash": ["🏄🏾", "👒", "👨🏽‍🎓", "🅰️", "👩🏾‍🍳", "🪀", "📩", "🐄", "⏳", "👸🏻", "🚣🏾‍♂️", "👩🏽‍🤝‍👩🏻", "📀", "🌒"], +} diff --git a/tests-functional/resources/test_data/mnemonic_24.py b/tests-functional/resources/test_data/mnemonic_24.py new file mode 100644 index 00000000000..fdc7c7d0704 --- /dev/null +++ b/tests-functional/resources/test_data/mnemonic_24.py @@ -0,0 +1,39 @@ +# Account data for mnemonic with 24 words + +accounts = [ + { + "address": "0x8a6d1f3b9f158f7274ca4be1a3f0056c86e2ccdb", + "public-key": ( + "0x04c25b359fab7fc8989d6325128b06dd9734b38d207dc2ab652e130a5d59852910fd6414694e1f5ce3a9cdd5c1" "f6bec9a425a57bae10c98ff4337adba3bf8c18bb" + ), + "path": "m/43'/60'/1581'/0'/0", + "prodPreferredChainIds": "1:10:42161:8453", + "operable": "fully", + "position": -1, + }, + { + "address": "0xf2d58ae5aa880f7c3f65d769296b1061c61e0955", + "public-key": ( + "0x04218096ceb5420c9b4cfa9d0187a057099540edff0aa5882b0a16b76fc8f0056d1a01930db4981f8885d00137" "c535740c04eec7ebe8bae7ef9fd98338fba31e04" + ), + "path": "m/44'/60'/0'/0/0", + "name": "Account 1", + "colorId": "primary", + "hidden": False, + "prodPreferredChainIds": "1:10:42161:8453", + "position": 0, + }, +] + +profile_data = { + "address": "0xb47386b0074a9ddfd979540f134915d1df8dc3d0", + "dapps-address": "0xf2d58ae5aa880f7c3f65d769296b1061c61e0955", + "eip1581-address": "0xf203f9c33afd10e2d3888289ad2cad81c4b017c4", + "key-uid": "0xcf119f28496e4123dd6d5a4936c5f595ee1a873b11ead5f275098456eb8777c4", + "name": "Selfassured Pesky Mayfly", + "public-key": ( + "0x04c25b359fab7fc8989d6325128b06dd9734b38d207dc2ab652e130a5d59852910fd6414694e1f5ce3a9cdd5c1f6be" "c9a425a57bae10c98ff4337adba3bf8c18bb" + ), + "wallet-root-address": "0x0410bd5715fdd8ccadede1d3131a9180a96e502c", + "emojiHash": ["👦🏻", "🕔", "🧜", "👩🏽‍🤝‍👨🏼", "👩🏿‍🔧", "🉐", "🧝🏿‍♂️", "🚣🏾‍♀️", "🫀", "🏄🏿", "🌘", "🤵🏼‍♀️", "🏄🏿‍♀️", "🎴"], +} diff --git a/tests-functional/tests/test_backup_mnemonic_and_restore.py b/tests-functional/tests/test_backup_mnemonic_and_restore.py new file mode 100644 index 00000000000..66f167b4c74 --- /dev/null +++ b/tests-functional/tests/test_backup_mnemonic_and_restore.py @@ -0,0 +1,173 @@ +import random +import string +from clients.status_backend import StatusBackend +from clients.signals import SignalType +import pytest + +from resources.constants import user_mnemonic_12, user_mnemonic_15, user_mnemonic_24, user_keycard_1 +from resources.utils import assert_response_attributes + + +@pytest.mark.create_account +@pytest.mark.rpc +class TestBackupMnemonicAndRestore: + + await_signals = [ + SignalType.MEDIASERVER_STARTED.value, + SignalType.NODE_STARTED.value, + SignalType.NODE_READY.value, + SignalType.NODE_LOGIN.value, + SignalType.NODE_LOGOUT.value, + ] + + @pytest.fixture(autouse=True) + def setup_cleanup(self, close_status_backend_containers): + """Automatically cleanup containers after each test""" + yield + + def test_profile_creation_and_mnemonics_backup(self): + # Create a new account container and initialize + account = StatusBackend(self.await_signals) + account.init_status_backend() + account.create_account_and_login() + account.wait_for_login() + + # Retrieve and verify the mnemonic + settings = account.settings_service.get_settings() + mnemonic = settings.get("result", {}).get("mnemonic", None) + assert mnemonic is not None + assert isinstance(mnemonic, str) + assert len(mnemonic.split()) == 12 # Basic check for mnemonic length + + def test_backup_account_and_restore_it_via_mnemonics(self): + # Create original account and backup mnemonic + original_account = StatusBackend(self.await_signals) + original_account.init_status_backend() + original_account.create_account_and_login() + original_account.wait_for_login() + original_get_settings_response = original_account.settings_service.get_settings() + original_settings = original_get_settings_response.get("result", {}) + mnemonic = original_settings.get("mnemonic", None) + assert mnemonic is not None + + user = user_mnemonic_12 + user.passphrase = mnemonic + + # Restore account in a new container + restored_account = StatusBackend(self.await_signals) + restored_account.init_status_backend() + restored_account.restore_account_and_login(user=user) + restored_account.wait_for_login() + + # Verify mnemonic is not exposed after restore + restored_get_settings_response = restored_account.settings_service.get_settings() + restored_settings = restored_get_settings_response.get("result", {}) + assert restored_settings.get("mnemonic", None) is None + + # Verify keys in the restored respone match the original respones + for key in [ + "address", + "compressedKey", + "current-user-status", + "dapps-address", + "eip1581-address", + "emojiHash", + "name", + "public-key", + "wallet-root-address", + ]: + assert original_settings.get(key) == restored_settings.get(key), f"Restored {key} doesn't match with original" + + # But some are different as expected + assert original_settings.get("installation-id") != restored_settings.get("installation-id"), "installation-id shouldn't match" + + @pytest.mark.parametrize( + "user_mnemonic", + [user_mnemonic_24, user_mnemonic_15, user_mnemonic_12], + ids=["mnemonic_24", "mnemonic_15", "mnemonic_12"], + ) + def test_restore_app_different_valid_size_mnemonics(self, user_mnemonic): + # Initialize backend client and restore account using user_mnemonic. + restored_account = StatusBackend(self.await_signals) + restored_account.init_status_backend() + restored_account.restore_account_and_login(user=user_mnemonic) + restored_account.wait_for_login() + + # Request getAccounts and check restoreed accounts attributes. + get_accounts_response = restored_account.accounts_service.get_accounts() + assert_response_attributes(get_accounts_response.get("result", {}), user_mnemonic.accounts) + + # Request getSettings and check restoreed profile data attributes. + get_settings_response = restored_account.settings_service.get_settings() + restored_settings = get_settings_response.get("result", {}) + assert_response_attributes(restored_settings, user_mnemonic.profile_data) + + assert restored_settings.get("mnemonic", None) is None + assert restored_settings.get("address", None) + + @pytest.mark.parametrize( + "mnemonic_size", + [1, 3, 7, 13, 29], + ) + def test_restore_with_arbitrary_size_mnemonics(self, mnemonic_size): + # Restore with an arbitrary length mnemonic + user = user_mnemonic_12 + user.passphrase = " ".join("".join(random.choice(string.ascii_lowercase) for _ in range(random.randint(2, 10))) for _ in range(mnemonic_size)) + + restored_account = StatusBackend(self.await_signals) + restored_account.init_status_backend() + restored_account.restore_account_and_login(user=user) + restored_account.wait_for_login() + + get_settings_response = restored_account.settings_service.get_settings() + restored_settings = get_settings_response.get("result", {}) + assert restored_settings.get("mnemonic", None) is None + assert restored_settings.get("address", None) + + def test_restore_with_mnemonic_with_special_chars(self): + # Restore with an mnemonic with special chars + user = user_mnemonic_12 + user.passphrase = "<>?`~!@#$%^&*()_+1 $fgdg ^&*()" + + restored_account = StatusBackend(self.await_signals) + restored_account.init_status_backend() + restored_account.restore_account_and_login(user=user) + restored_account.wait_for_login() + get_settings_response = restored_account.settings_service.get_settings() + restored_settings = get_settings_response.get("result", {}) + assert restored_settings.get("mnemonic", None) is None + assert restored_settings.get("address", None) + + def test_restore_with_empty_mnemonic(self): + # Restore with empty mnemonic isn't allowed + user = user_mnemonic_12 + + restored_account = StatusBackend(self.await_signals) + restored_account.init_status_backend() + restored_account._set_display_name() + data = restored_account._create_account_request(user) + data["mnemonic"] = "" + restored_account_response = restored_account.api_request("RestoreAccountAndLogin", data) + assert restored_account_response.json().get("error") == "restore-account: no mnemonic or keycard is set" + + def test_restore_with_both_mnemonic_and_keycard(self): + # Restore with both keycard and mnemonic isn't allowed + user = user_mnemonic_12 + restored_account = StatusBackend(self.await_signals) + restored_account.init_status_backend() + restored_account._set_display_name() + data = restored_account._create_account_request(user) + data["mnemonic"] = user.passphrase + data["keycard"] = user_keycard_1 + restored_account_response = restored_account.api_request("RestoreAccountAndLogin", data) + assert restored_account_response.json().get("error") == "restore-account: both mnemonic and keycard info are set" + + def test_restored_on_existing_restored_account_fails(self): + user = user_mnemonic_12 + restored_account = StatusBackend(self.await_signals) + restored_account.init_status_backend() + restored_account.restore_account_and_login(user=user) + restored_account.wait_for_login() + restored_account.restore_account_and_login(user=user) + signal = restored_account.wait_for_signal(SignalType.NODE_LOGIN.value) + assert signal.get("event").get("error") == "UNIQUE constraint failed: settings.synthetic_id" From c0100c2f50cafadde5fd24d82272973fe5f2576d Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Fri, 1 Aug 2025 13:54:01 -0400 Subject: [PATCH 03/15] chore(backup)_: enable local backup feature flag Part of https://github.com/status-im/status-desktop/issues/18400 --- node/get_status_node.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/node/get_status_node.go b/node/get_status_node.go index 7ed1d633732..628d2306d4a 100644 --- a/node/get_status_node.go +++ b/node/get_status_node.go @@ -257,9 +257,8 @@ func (n *StatusNode) StartLocalBackup() error { n.localBackup, err = backup.NewController(backup.BackupConfig{ PrivateKey: crypto.Keccak256(crypto.FromECDSA(privateKey)), FileNameGetter: filenameGetter, - // TODO set to true to enable the local backup - BackupEnabled: false, - Interval: time.Minute * 30, + BackupEnabled: true, + Interval: time.Minute * 30, }, n.logger.Named("LocalBackup")) if err != nil { return err From f0fa5caed20c72d29de6c31234cb11d7f03b8643 Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Tue, 5 Aug 2025 12:14:23 -0400 Subject: [PATCH 04/15] fix(backup)_: fix missing image in group events --- protocol/messenger_backup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/protocol/messenger_backup.go b/protocol/messenger_backup.go index c3f4365f774..263f28f35c9 100644 --- a/protocol/messenger_backup.go +++ b/protocol/messenger_backup.go @@ -390,6 +390,7 @@ func (m *Messenger) backupChats(ctx context.Context, clock uint64) []*protobuf.B From: membershipUpdate.From, RawPayload: membershipUpdate.RawPayload, Color: membershipUpdate.Color, + Image: membershipUpdate.Image, } } } From a184be6f7b87334f045b2c23b8c3b39ec4cbf6fe Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Tue, 5 Aug 2025 12:22:49 -0400 Subject: [PATCH 05/15] fix(backup)_: ignore already backed up watch accounts --- services/wallet/service.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/wallet/service.go b/services/wallet/service.go index e24e930f72b..9ccfec7617e 100644 --- a/services/wallet/service.go +++ b/services/wallet/service.go @@ -574,7 +574,8 @@ func (s *Service) handleSyncWatchOnlyAccount(message *protobuf.SyncAccount) (*ac if dbAccount != nil { if message.Clock <= dbAccount.Clock { - return nil, ErrTryingToStoreOldWalletAccount + // ignore this old message + return nil, nil } if message.Removed { From be02ece3ec67170c21b30ab37eb0991a5870a3d8 Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Tue, 5 Aug 2025 12:23:45 -0400 Subject: [PATCH 06/15] fix(backup)_: process all backups before throwing an error --- node/backup/core.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/node/backup/core.go b/node/backup/core.go index d608762c650..52187618136 100644 --- a/node/backup/core.go +++ b/node/backup/core.go @@ -3,6 +3,7 @@ package backup import ( "bytes" "encoding/gob" + "errors" "fmt" "github.com/status-im/status-go/eth-node/crypto" @@ -78,17 +79,18 @@ func (b *core) exportBackup() (map[string][]byte, error) { } func (b *core) importBackup(data map[string][]byte) error { + var errs []error for name, provider := range b.backupProviders { raw, ok := data[name] if !ok { continue } if err := provider.ImportBackup(raw); err != nil { - return fmt.Errorf("importBackup %q failed: %w", name, err) + errs = append(errs, fmt.Errorf("importBackup %q failed: %w", name, err)) } } - return nil + return errors.Join(errs...) } func marshal(data map[string][]byte) ([]byte, error) { From 7586c41ff8b89981f0797a9c5e64a1004eaceda8 Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Fri, 8 Aug 2025 12:06:12 -0400 Subject: [PATCH 07/15] fix(backup)_: fix account and contact images from backup not having localUrl Fixes https://github.com/status-im/status-desktop/issues/18533 --- protocol/messenger_backup_handler.go | 7 +++++++ protocol/messenger_handler.go | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/protocol/messenger_backup_handler.go b/protocol/messenger_backup_handler.go index 580d005f736..28fd0419179 100644 --- a/protocol/messenger_backup_handler.go +++ b/protocol/messenger_backup_handler.go @@ -376,6 +376,13 @@ func (m *Messenger) handleBackedUpProfile(message *protobuf.BackedUpProfile, bac if err != nil { return err } + + if m.httpServer != nil { + for j, image := range idImages { + idImages[j].LocalURL = m.httpServer.MakeAccountImageURL(message.KeyUid, image.Name, image.Clock) + } + } + response.SetImages(idImages) } } diff --git a/protocol/messenger_handler.go b/protocol/messenger_handler.go index 89fc917e835..a2229ea98d9 100644 --- a/protocol/messenger_handler.go +++ b/protocol/messenger_handler.go @@ -754,6 +754,10 @@ func (m *Messenger) HandleSyncInstallationContactV2(state *ReceivedMessageState, state.Response.AddChat(chat) } + if err = m.updateContactImagesURL(contact); err != nil { + return err + } + state.ModifiedContacts.Store(contact.ID, true) state.AllContacts.Store(contact.ID, contact) } From 709900f25bc259af0d07537a525f4cc5bf4a81cf Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Fri, 15 Aug 2025 11:24:33 -0400 Subject: [PATCH 08/15] chore(backup): remove waku backup and keep only local backup Needed for https://github.com/status-im/status-desktop/issues/18123 --- api/backend.go | 29 +- api/backend_test.go | 980 ++++------- api/defaults.go | 2 - api/geth_backend.go | 1486 ++++++----------- ...5274768573_drop_waku_backup_setings.up.sql | 2 + multiaccounts/accounts/database.go | 1 - multiaccounts/settings/columns.go | 17 - multiaccounts/settings/database.go | 42 +- .../settings/database_settings_manager.go | 5 - multiaccounts/settings/database_test.go | 1 - multiaccounts/settings/structs.go | 3 - params/config.go | 3 - protocol/activity_center.go | 8 +- protocol/common/feature_flags.go | 3 - ...es_messenger_shared_member_address_test.go | 64 +- .../communities_messenger_signers_test.go | 2 +- ...nities_messenger_token_permissions_test.go | 322 ++-- protocol/messenger.go | 524 ++---- protocol/messenger_backup.go | 560 ------- protocol/messenger_backup_handler.go | 329 +--- protocol/messenger_backup_test.go | 1203 ------------- protocol/messenger_browsers_test.go | 10 +- protocol/messenger_builder_test.go | 124 +- protocol/messenger_config.go | 145 +- protocol/messenger_handler.go | 316 ++-- protocol/messenger_local_backup.go | 232 ++- protocol/messenger_mailserver.go | 161 +- protocol/messenger_mailserver_cycle.go | 2 +- protocol/messenger_sync_contact.go | 45 + protocol/messenger_testing_utils.go | 1 - .../application_metadata_message.proto | 2 +- protocol/protobuf/sync_settings.proto | 5 - protocol/requests/restore_account.go | 2 - server/pairing/sync_device_test.go | 1 - services/ext/api.go | 12 +- services/ext/service.go | 3 - services/ext/signal.go | 5 - signal/events_shhext.go | 8 - .../resources/test_data/mnemonic_12.py | 1 - 39 files changed, 1860 insertions(+), 4801 deletions(-) create mode 100644 appdatabase/migrations/sql/1755274768573_drop_waku_backup_setings.up.sql delete mode 100644 protocol/messenger_backup.go delete mode 100644 protocol/messenger_backup_test.go create mode 100644 protocol/messenger_sync_contact.go diff --git a/api/backend.go b/api/backend.go index 53486502f0b..ecf72e0b682 100644 --- a/api/backend.go +++ b/api/backend.go @@ -5,11 +5,12 @@ import ( signercore "github.com/ethereum/go-ethereum/signer/core/apitypes" - "github.com/status-im/status-go/eth-node/types" + accsmanagementtypes "github.com/status-im/status-go/accounts-management/types" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/multiaccounts" - "github.com/status-im/status-go/multiaccounts/accounts" "github.com/status-im/status-go/multiaccounts/settings" "github.com/status-im/status-go/params" + "github.com/status-im/status-go/protocol/requests" "github.com/status-im/status-go/services/personal" "github.com/status-im/status-go/services/typeddata" "github.com/status-im/status-go/services/wallet/wallettypes" @@ -17,23 +18,32 @@ import ( // StatusBackend defines the contract for the Status.im service type StatusBackend interface { - // IsNodeRunning() bool // NOTE: Only used in tests - StartNode(config *params.NodeConfig) error + StartNode(config *params.NodeConfig) error // NOTE: Only used in canary StartNodeWithKey(acc multiaccounts.Account, password string, keyHex string, conf *params.NodeConfig) error StartNodeWithAccount(acc multiaccounts.Account, password string, conf *params.NodeConfig, chatKey *ecdsa.PrivateKey) error - StartNodeWithAccountAndInitialConfig(account multiaccounts.Account, password string, settings settings.Settings, conf *params.NodeConfig, subaccs []*accounts.Account, chatKey *ecdsa.PrivateKey) error + StartNodeWithChatKeyOrMnemonic( + request *requests.CreateAccount, + mnemonic string, // empty mnemonic is used for keycard account, not empty for regular account + keycardData *requests.KeycardData, // nil for regular account, not nil for keycard account + restoreAccount bool, + ) (*multiaccounts.Account, error) + StartNodeWithAccountAndInitialConfig( + multiAccount *multiaccounts.Account, + password string, + settings settings.Settings, + nodecfg *params.NodeConfig, + keypair *accsmanagementtypes.Keypair, + chatKey *ecdsa.PrivateKey) error StopNode() error - // RestartNode() error // NOTE: Only used in tests GetNodeConfig() (*params.NodeConfig, error) UpdateRootDataDir(datadir string) - // SelectAccount(loginParams account.LoginParams) error + SelectAccount(loginParams LoginParams, privateKey *ecdsa.PrivateKey) error OpenAccounts() error GetAccounts() ([]multiaccounts.Account, error) LocalPairingStarted() error - // SaveAccount(account multiaccounts.Account) error - SaveAccountAndStartNodeWithKey(acc multiaccounts.Account, password string, settings settings.Settings, conf *params.NodeConfig, subaccs []*accounts.Account, keyHex string) error + SaveAccount(account multiaccounts.Account) error Recover(rpcParams personal.RecoverParams) (types.Address, error) Logout() error @@ -42,7 +52,6 @@ type StatusBackend interface { HashTransaction(sendArgs wallettypes.SendTxArgs) (wallettypes.SendTxArgs, types.Hash, error) HashTypedData(typed typeddata.TypedData) (types.Hash, error) HashTypedDataV4(typed signercore.TypedData) (types.Hash, error) - ResetChainData() error SendTransaction(sendArgs wallettypes.SendTxArgs, password string) (hash types.Hash, err error) SendTransactionWithChainID(chainID uint64, sendArgs wallettypes.SendTxArgs, password string) (hash types.Hash, err error) SendTransactionWithSignature(sendArgs wallettypes.SendTxArgs, sig []byte) (hash types.Hash, err error) diff --git a/api/backend_test.go b/api/backend_test.go index 3bcfbce75c6..54efda8d072 100644 --- a/api/backend_test.go +++ b/api/backend_test.go @@ -4,14 +4,11 @@ import ( "context" "crypto/sha256" "database/sql" - "encoding/hex" "encoding/json" "fmt" - "io/ioutil" "math/rand" "os" "path" - "path/filepath" "strings" "sync" "testing" @@ -24,51 +21,35 @@ import ( "github.com/brianvoe/gofakeit/v6" - "github.com/status-im/status-go/api/common" + accsmanagement "github.com/status-im/status-go/accounts-management" + accscommon "github.com/status-im/status-go/accounts-management/common" + accsmanagementcommon "github.com/status-im/status-go/accounts-management/common" + "github.com/status-im/status-go/accounts-management/generator" + "github.com/status-im/status-go/accounts-management/keystore" + accsmanagementtypes "github.com/status-im/status-go/accounts-management/types" "github.com/status-im/status-go/appdatabase" "github.com/status-im/status-go/connection" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" - "github.com/status-im/status-go/internal/security" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/multiaccounts" "github.com/status-im/status-go/multiaccounts/accounts" "github.com/status-im/status-go/multiaccounts/settings" "github.com/status-im/status-go/node" "github.com/status-im/status-go/params" + "github.com/status-im/status-go/pkg/security" "github.com/status-im/status-go/protocol/requests" "github.com/status-im/status-go/protocol/tt" "github.com/status-im/status-go/rpc" "github.com/status-im/status-go/services/typeddata" "github.com/status-im/status-go/services/wallet" walletservice "github.com/status-im/status-go/services/wallet" - "github.com/status-im/status-go/services/wallet/wallettypes" + "github.com/status-im/status-go/services/wallet/common" "github.com/status-im/status-go/signal" - "github.com/status-im/status-go/sqlite" "github.com/status-im/status-go/t/helpers" "github.com/status-im/status-go/t/utils" "github.com/status-im/status-go/walletdatabase" ) -var ( - networks = json.RawMessage("{}") - testSettings = settings.Settings{ - Address: types.HexToAddress("0xeC540f3745Ff2964AFC1171a5A0DD726d1F6B472"), - DisplayName: "UserDisplayName", - CurrentNetwork: "mainnet_rpc", - DappsAddress: types.HexToAddress("0xe1300f99fDF7346986CbC766903245087394ecd0"), - EIP1581Address: types.HexToAddress("0xe1DDDE9235a541d1344550d969715CF43982de9f"), - InstallationID: "d3efcff6-cffa-560e-a547-21d3858cbc51", - KeyUID: "0x4e8129f3edfc004875be17bf468a784098a9f69b53c095be1f52deff286935ab", - LatestDerivedPath: 0, - Name: "Jittery Cornflowerblue Kingbird", - Networks: &networks, - PhotoPath: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAIAAACRXR/mAAAAjklEQVR4nOzXwQmFMBAAUZXUYh32ZB32ZB02sxYQQSZGsod55/91WFgSS0RM+SyjA56ZRZhFmEWYRRT6h+M6G16zrxv6fdJpmUWYRbxsYr13dKfanpN0WmYRZhGzXz6AWYRZRIfbaX26fT9Jk07LLMIsosPt9I/dTDotswizCG+nhFmEWYRZhFnEHQAA///z1CFkYamgfQAAAABJRU5ErkJggg==", - PreviewPrivacy: false, - PublicKey: "0x04211fe0f69772ecf7eb0b5bfc7678672508a9fb01f2d699096f0d59ef7fe1a0cb1e648a80190db1c0f5f088872444d846f2956d0bd84069f3f9f69335af852ac0", - SigningPhrase: "yurt joey vibe", - WalletRootAddress: types.HexToAddress("0xeB591fd819F86D0A6a2EF2Bcb94f77807a7De1a6")} -) - func setupTestDB() (*sql.DB, func() error, error) { return helpers.SetupTestSQLDB(appdatabase.DbInitializer{}, "tests") } @@ -78,7 +59,7 @@ func setupTestWalletDB() (*sql.DB, func() error, error) { } func setupTestMultiDB() (*multiaccounts.Database, func() error, error) { - tmpfile, err := ioutil.TempFile("", "tests") + tmpfile, err := os.CreateTemp("", "tests") if err != nil { return nil, nil, err } @@ -121,6 +102,12 @@ func setupGethStatusBackend() (*GethStatusBackend, func() error, func() error, f return backend, stop1, stop2, stop3, err } +func handleError(t *testing.T, err error) { + if err != nil { + t.Logf("deferred function error: '%s'", err) + } +} + func TestBackendStartNodeConcurrently(t *testing.T) { utils.Init() @@ -147,7 +134,7 @@ func TestBackendStartNodeConcurrently(t *testing.T) { config, err := utils.MakeTestNodeConfig(params.StatusChainNetworkID) require.NoError(t, err) - require.NoError(t, backend.AccountManager().InitKeystore(config.KeyStoreDir)) + count := 2 resultCh := make(chan error) @@ -202,8 +189,8 @@ func TestBackendRestartNodeConcurrently(t *testing.T) { config, err := utils.MakeTestNodeConfig(params.StatusChainNetworkID) require.NoError(t, err) - count := 3 - require.NoError(t, backend.AccountManager().InitKeystore(config.KeyStoreDir)) + const count = 3 + require.NoError(t, backend.StartNode(config)) defer func() { require.NoError(t, backend.StopNode()) @@ -222,8 +209,6 @@ func TestBackendRestartNodeConcurrently(t *testing.T) { wg.Wait() } -// TODO(adam): add concurrent tests for ResetChainData() - func TestBackendGettersConcurrently(t *testing.T) { utils.Init() @@ -250,7 +235,7 @@ func TestBackendGettersConcurrently(t *testing.T) { config, err := utils.MakeTestNodeConfig(params.StatusChainNetworkID) require.NoError(t, err) - require.NoError(t, backend.AccountManager().InitKeystore(config.KeyStoreDir)) + err = backend.StartNode(config) require.NoError(t, err) defer func() { @@ -267,13 +252,13 @@ func TestBackendGettersConcurrently(t *testing.T) { wg.Add(1) go func() { - assert.NotNil(t, backend.AccountManager()) + assert.NotNil(t, backend.AccountsManager()) wg.Done() }() wg.Add(1) go func() { - assert.NotNil(t, backend.personalAPI) + assert.NotNil(t, backend.signer) wg.Done() }() @@ -357,8 +342,6 @@ func TestBackendCallRPCConcurrently(t *testing.T) { config, err := utils.MakeTestNodeConfig(params.StatusChainNetworkID) require.NoError(t, err) - require.NoError(t, backend.AccountManager().InitKeystore(config.KeyStoreDir)) - count := 3 err = backend.StartNode(config) require.NoError(t, err) @@ -368,6 +351,7 @@ func TestBackendCallRPCConcurrently(t *testing.T) { var wg sync.WaitGroup + const count = 3 for i := 0; i < count; i++ { wg.Add(1) go func(idx int) { @@ -453,7 +437,7 @@ func TestBlockedRPCMethods(t *testing.T) { config, err := utils.MakeTestNodeConfig(params.StatusChainNetworkID) require.NoError(t, err) - require.NoError(t, backend.AccountManager().InitKeystore(config.KeyStoreDir)) + err = backend.StartNode(config) require.NoError(t, err) defer func() { require.NoError(t, backend.StopNode()) }() @@ -513,13 +497,6 @@ func TestStartStopMultipleTimes(t *testing.T) { config, err := utils.MakeTestNodeConfig(params.StatusChainNetworkID) require.NoError(t, err) - require.NoError(t, backend.AccountManager().InitKeystore(config.KeyStoreDir)) - config.NoDiscovery = false - // doesn't have to be running. just any valid enode to bypass validation. - config.ClusterConfig.BootNodes = []string{ - "enode://e8a7c03b58911e98bbd66accb2a55d57683f35b23bf9dfca89e5e244eb5cc3f25018b4112db507faca34fb69ffb44b362f79eda97a669a8df29c72e654416784@0.0.0.0:30404", - } - require.NoError(t, err) require.NoError(t, backend.StartNode(config)) require.NoError(t, backend.StopNode()) require.NoError(t, backend.StartNode(config)) @@ -552,7 +529,7 @@ func TestHashTypedData(t *testing.T) { config, err := utils.MakeTestNodeConfig(params.StatusChainNetworkID) require.NoError(t, err) - require.NoError(t, backend.AccountManager().InitKeystore(config.KeyStoreDir)) + err = backend.StartNode(config) require.NoError(t, err) defer func() { @@ -597,257 +574,210 @@ func TestHashTypedData(t *testing.T) { func TestBackendGetVerifiedAccount(t *testing.T) { utils.Init() - password := "test" - backend, defers, err := setupWalletTest(t, password) + testContext := setupTestContext(t, testPassword, true, true, false) + + err := testContext.backend.StartNode(testContext.config) require.NoError(t, err) - defer defers() + + defer func() { + require.NoError(t, testContext.backend.StopNode()) + }() t.Run("AccountDoesntExist", func(t *testing.T) { pkey, err := gethcrypto.GenerateKey() require.NoError(t, err) address := gethcrypto.PubkeyToAddress(pkey.PublicKey) - key, err := backend.getVerifiedWalletAccount(address.String(), password) - require.EqualError(t, err, wallettypes.ErrAccountDoesntExist.Error()) + key, err := testContext.backend.getVerifiedWalletAccount(address.String(), testPassword) + require.EqualError(t, err, accsmanagement.ErrAccountDoesNotExist.Error()) require.Nil(t, key) }) t.Run("PasswordDoesntMatch", func(t *testing.T) { - pkey, err := crypto.GenerateKey() + pkey, err := gethcrypto.GenerateKey() require.NoError(t, err) - address := crypto.PubkeyToAddress(pkey.PublicKey) - keyUIDHex := sha256.Sum256(gethcrypto.FromECDSAPub(&pkey.PublicKey)) - keyUID := types.EncodeHex(keyUIDHex[:]) - - db, err := accounts.NewDB(backend.appDB) + privateKeyHex := types.EncodeHex(gethcrypto.FromECDSA(pkey)) + address := gethcrypto.PubkeyToAddress(pkey.PublicKey) + _, err = testContext.backend.AccountsManager().CreateKeypairFromPrivateKeyAndStore(privateKeyHex, testPassword, "private key keypair", &accsmanagementtypes.AccountCreationDetails{ + Path: accsmanagementcommon.PathMaster, + }, 0) require.NoError(t, err) - _, err = backend.AccountManager().ImportAccount(pkey, password) - require.NoError(t, err) - require.NoError(t, db.SaveOrUpdateKeypair(&accounts.Keypair{ - KeyUID: keyUID, - Name: "private key keypair", - Type: accounts.KeypairTypeKey, - Accounts: []*accounts.Account{ - &accounts.Account{ - Address: address, - KeyUID: keyUID, - }, - }, - })) - key, err := backend.getVerifiedWalletAccount(address.String(), "wrong-password") - require.EqualError(t, err, "could not decrypt key with given password") + + key, err := testContext.backend.getVerifiedWalletAccount(address.String(), "wrong-password") + require.EqualError(t, err, keystore.ErrIncorrectPasswordProvided.Error()) require.Nil(t, key) }) - t.Run("PartialAccount", func(t *testing.T) { + t.Run("PartialAccount", func(t *testing.T) { // This is mobile app specific test and can be removed // Create a derived wallet account without storing the keys - db, err := accounts.NewDB(backend.appDB) + db, err := accounts.NewDB(testContext.backend.appDB) + require.NoError(t, err) + + err = db.CreateSettings(testContext.settings, *testContext.config) require.NoError(t, err) - newPath := "m/0" + walletRootAddress, err := db.GetWalletRootAddress() require.NoError(t, err) - walletInfo, err := backend.AccountManager().AccountsGenerator().LoadAccount(walletRootAddress.String(), password) + // Store keystore file for the wallet root address + masterAcc, derivedAccs, err := testContext.backend.AccountsManager().StoreKeystoreFilesForMnemonic(testContext.mnemonic, testPassword, + []string{accscommon.PathWalletRoot}) require.NoError(t, err) - derivedInfos, err := backend.AccountManager().AccountsGenerator().DeriveAddresses(walletInfo.ID, []string{newPath}) + + walletRootGeneratedAccount := derivedAccs[accscommon.PathWalletRoot] + require.Equal(t, walletRootAddress, walletRootGeneratedAccount.Address()) + + // check the number of wallet addresses in the db + walletAddresses, err := db.GetWalletAddresses() require.NoError(t, err) - derivedInfo := derivedInfos[newPath] - - keypair := &accounts.Keypair{ - KeyUID: walletInfo.KeyUID, - Name: "profile keypair", - Type: accounts.KeypairTypeProfile, - Accounts: []*accounts.Account{ - &accounts.Account{ - Address: types.HexToAddress(derivedInfo.Address), - KeyUID: walletInfo.KeyUID, - Type: accounts.AccountTypeGenerated, - PublicKey: types.Hex2Bytes(derivedInfo.PublicKey), - Path: newPath, - Wallet: false, - Name: "PartialAccount", - }, + require.Equal(t, 2, len(walletAddresses)) // should be 1, but because of the tests `Run` before this one it's 2 + + // Create a new wallet account and store it to db only, without storing the keystore file + derivedWalletAcc1, err := generator.DeriveChildFromAccount(masterAcc, accscommon.CustomWalletPath1) + require.NoError(t, err) + + err = db.SaveOrUpdateAccounts([]*accsmanagementtypes.Account{ + { + Address: derivedWalletAcc1.Address(), + KeyUID: masterAcc.KeyUID(), + Path: accscommon.CustomWalletPath1, }, - } - require.NoError(t, db.SaveOrUpdateKeypair(keypair)) + }, false) + require.NoError(t, err) - // With partial account we need to dynamically generate private key - key, err := backend.getVerifiedWalletAccount(keypair.Accounts[0].Address.Hex(), password) + // check the number of wallet addresses in the db, to ensure that the account was saved + walletAddresses, err = db.GetWalletAddresses() require.NoError(t, err) - require.Equal(t, keypair.Accounts[0].Address, key.Address) + require.Equal(t, 3, len(walletAddresses)) // should be 2, but because of the tests `Run` before this one it's 3 + + // try to load the account, it should fail because the account is not in the keystore, just in the db + loadedWalletAcc1, err := testContext.backend.AccountsManager().LoadAccount(derivedWalletAcc1.Address(), testPassword) + require.Error(t, err) + // Check if the error contains the expected message (new structured error format includes context) + require.Contains(t, err.Error(), "keystore file is missing") + require.Nil(t, loadedWalletAcc1) + + // try to get verified wallet account, it should generate a keystore for the wallet account from the wallet root address + verifiedWalletAcc1, err := testContext.backend.AccountsManager().GetVerifiedWalletAccount(derivedWalletAcc1.Address(), + testPassword) + require.NoError(t, err) + + loadedWalletAcc1, err = testContext.backend.AccountsManager().LoadAccount(derivedWalletAcc1.Address(), testPassword) + require.NoError(t, err) + + // derive, verified and loaded wallet account should be the same + require.Equal(t, derivedWalletAcc1.Address(), verifiedWalletAcc1.Address()) + require.Equal(t, derivedWalletAcc1.Address(), loadedWalletAcc1.Address()) }) t.Run("Success", func(t *testing.T) { pkey, err := crypto.GenerateKey() require.NoError(t, err) + privateKeyHex := types.EncodeHex(crypto.FromECDSA(pkey)) address := crypto.PubkeyToAddress(pkey.PublicKey) keyUIDHex := sha256.Sum256(gethcrypto.FromECDSAPub(&pkey.PublicKey)) keyUID := types.EncodeHex(keyUIDHex[:]) - db, err := accounts.NewDB(backend.appDB) + db, err := accounts.NewDB(testContext.backend.appDB) require.NoError(t, err) - defer db.Close() - _, err = backend.AccountManager().ImportAccount(pkey, password) + defer func() { + handleError(t, db.Close()) + }() + + keypair, err := testContext.backend.AccountsManager().CreateKeypairFromPrivateKeyAndStore(privateKeyHex, testPassword, + "private key keypair", &accsmanagementtypes.AccountCreationDetails{ + Path: accsmanagementcommon.PathMaster, + }, 0) require.NoError(t, err) - require.NoError(t, db.SaveOrUpdateKeypair(&accounts.Keypair{ - KeyUID: keyUID, - Name: "private key keypair", - Type: accounts.KeypairTypeKey, - Accounts: []*accounts.Account{ - &accounts.Account{ - Address: address, - KeyUID: keyUID, - }, - }, - })) - key, err := backend.getVerifiedWalletAccount(address.String(), password) + + require.Equal(t, keypair.KeyUID, keyUID) + require.Len(t, keypair.Accounts, 1) + require.Equal(t, keypair.Accounts[0].Address, address) + + acc, err := testContext.backend.getVerifiedWalletAccount(address.String(), testPassword) require.NoError(t, err) - require.Equal(t, address, key.Address) + require.Equal(t, address, acc.Address()) }) } func TestRuntimeLogLevelIsNotWrittenToDatabase(t *testing.T) { utils.Init() - b := NewGethStatusBackend(tt.MustCreateTestLogger()) - - chatKey, err := gethcrypto.GenerateKey() - require.NoError(t, err) - walletKey, err := gethcrypto.GenerateKey() - require.NoError(t, err) - keyUIDHex := sha256.Sum256(gethcrypto.FromECDSAPub(&chatKey.PublicKey)) - keyUID := types.EncodeHex(keyUIDHex[:]) - main := multiaccounts.Account{ - KeyUID: keyUID, - } - - tmpdir := t.TempDir() + testContext := setupTestContext(t, testPassword, false, false, false) json := `{ "NetworkId": 3, - "DataDir": "` + tmpdir + `", - "KeyStoreDir": "` + tmpdir + `", - "KeycardPairingDataFile": "` + path.Join(tmpdir, "keycard/pairings.json") + `", + "DataDir": "` + testContext.config.DataDir + `", + "KeycardPairingDataFile": "` + path.Join(testContext.config.DataDir, "keycard/pairings.json") + `", "NoDiscovery": true, "TorrentConfig": { "Port": 9025, "Enabled": false, - "DataDir": "` + tmpdir + `/archivedata", - "TorrentDir": "` + tmpdir + `/torrents" + "DataDir": "` + testContext.config.DataDir + `/archivedata", + "TorrentDir": "` + testContext.config.DataDir + `/torrents" }, "RuntimeLogLevel": "INFO", "LogLevel": "DEBUG" }` - conf, err := params.NewConfigFromJSON(json) + newConf, err := params.NewConfigFromJSON(json) require.NoError(t, err) - require.Equal(t, "INFO", conf.RuntimeLogLevel) - keyhex := hex.EncodeToString(gethcrypto.FromECDSA(chatKey)) - - require.NoError(t, b.AccountManager().InitKeystore(conf.KeyStoreDir)) - b.UpdateRootDataDir(conf.DataDir) - require.NoError(t, b.OpenAccounts()) - require.NotNil(t, b.statusNode.HTTPServer()) - - address := crypto.PubkeyToAddress(walletKey.PublicKey) + require.Equal(t, "INFO", newConf.RuntimeLogLevel) - settings := testSettings - settings.KeyUID = keyUID - settings.Address = crypto.PubkeyToAddress(walletKey.PublicKey) + require.NoError(t, testContext.backend.OpenAccounts()) + require.NotNil(t, testContext.backend.statusNode.HTTPServer()) - chatPubKey := crypto.FromECDSAPub(&chatKey.PublicKey) - require.NoError(t, b.SaveAccountAndStartNodeWithKey(main, "test-pass", settings, conf, - []*accounts.Account{ - {Address: address, KeyUID: keyUID, Wallet: true}, - {Address: crypto.PubkeyToAddress(chatKey.PublicKey), KeyUID: keyUID, Chat: true, PublicKey: chatPubKey}}, keyhex)) - require.NoError(t, b.Logout()) - require.NoError(t, b.StopNode()) - - require.NoError(t, b.StartNodeWithKey(main, "test-pass", keyhex, conf)) - defer func() { - assert.NoError(t, b.Logout()) - assert.NoError(t, b.StopNode()) - }() - - c, err := b.GetNodeConfig() + err = testContext.backend.ensureDBsOpened(*testContext.multiAcc, testPassword) require.NoError(t, err) - require.Equal(t, "", c.RuntimeLogLevel) - require.Equal(t, "DEBUG", c.LogLevel) -} - -func TestLoginWithKey(t *testing.T) { - utils.Init() - - b := NewGethStatusBackend(tt.MustCreateTestLogger()) - chatKey, err := gethcrypto.GenerateKey() - require.NoError(t, err) - walletKey, err := gethcrypto.GenerateKey() - require.NoError(t, err) - keyUIDHex := sha256.Sum256(gethcrypto.FromECDSAPub(&chatKey.PublicKey)) - keyUID := types.EncodeHex(keyUIDHex[:]) - main := multiaccounts.Account{ - KeyUID: keyUID, + request := &requests.CreateAccount{ + RootDataDir: testContext.config.DataDir, + Password: testPassword, + KdfIterations: 1, } - tmpdir := t.TempDir() - conf, err := params.NewNodeConfig(tmpdir, 1777) - require.NoError(t, err) - keyhex := hex.EncodeToString(gethcrypto.FromECDSA(chatKey)) - - require.NoError(t, b.AccountManager().InitKeystore(conf.KeyStoreDir)) - b.UpdateRootDataDir(conf.DataDir) - require.NoError(t, b.OpenAccounts()) - require.NotNil(t, b.statusNode.HTTPServer()) - address := crypto.PubkeyToAddress(walletKey.PublicKey) - - settings := testSettings - settings.KeyUID = keyUID - settings.Address = crypto.PubkeyToAddress(walletKey.PublicKey) + _, err = testContext.backend.StartNodeWithChatKeyOrMnemonic( + request, + testContext.mnemonic, + nil, + false, + ) + require.NoError(t, err) - chatPubKey := crypto.FromECDSAPub(&chatKey.PublicKey) - require.NoError(t, b.SaveAccountAndStartNodeWithKey(main, "test-pass", settings, conf, - []*accounts.Account{ - {Address: address, KeyUID: keyUID, Wallet: true}, - {Address: crypto.PubkeyToAddress(chatKey.PublicKey), KeyUID: keyUID, Chat: true, PublicKey: chatPubKey}}, keyhex)) - require.NoError(t, b.Logout()) - require.NoError(t, b.StopNode()) + require.NoError(t, testContext.backend.Logout()) + require.NoError(t, testContext.backend.StopNode()) - require.NoError(t, b.AccountManager().InitKeystore(conf.KeyStoreDir)) - b.UpdateRootDataDir(conf.DataDir) - require.NoError(t, b.OpenAccounts()) + chatPrivKey := strings.TrimPrefix(testContext.chatPrivateKey, "0x") + require.NoError(t, testContext.backend.StartNodeWithKey(*testContext.multiAcc, testPassword, chatPrivKey, newConf)) - require.NoError(t, b.StartNodeWithKey(main, "test-pass", keyhex, conf)) defer func() { - assert.NoError(t, b.Logout()) - assert.NoError(t, b.StopNode()) + assert.NoError(t, testContext.backend.Logout()) + assert.NoError(t, testContext.backend.StopNode()) }() - extkey, err := b.accountManager.SelectedChatAccount() - require.NoError(t, err) - require.Equal(t, crypto.PubkeyToAddress(chatKey.PublicKey), extkey.Address) - activeAccount, err := b.GetActiveAccount() + c, err := testContext.backend.GetNodeConfig() require.NoError(t, err) - require.NotNil(t, activeAccount.ColorHash) + require.Equal(t, "", c.RuntimeLogLevel) + require.Equal(t, "DEBUG", c.LogLevel) } func TestLoginAccount(t *testing.T) { utils.Init() - password := "some-password" - tmpdir := t.TempDir() - nameserver := "8.8.8.8" - b := NewGethStatusBackend(tt.MustCreateTestLogger()) + testContext := setupTestContext(t, testPassword, false, false, false) + + nameserver := "8.8.8.8" createAccountRequest := &requests.CreateAccount{ DisplayName: "some-display-name", CustomizationColor: "#ffffff", - Password: password, - RootDataDir: tmpdir, - LogFilePath: tmpdir + "/log", + Password: testPassword, + RootDataDir: testContext.config.DataDir, + LogFilePath: testContext.config.DataDir + "/log", WakuV2Nameserver: &nameserver, WakuV2Fleet: "status.staging", } + c := make(chan interface{}, 10) signal.SetMobileSignalHandler(func(data []byte) { if strings.Contains(string(data), signal.EventLoggedIn) { @@ -865,23 +795,19 @@ func TestLoginAccount(t *testing.T) { } } - acc, err := b.CreateAccountAndLogin(createAccountRequest) - require.NoError(t, err) - require.Equal(t, nameserver, b.config.WakuV2Config.Nameserver) - - accountsDB, err := b.accountsDB() - require.NoError(t, err) - backupFecthed, err := accountsDB.BackupFetched() + acc, err := testContext.backend.CreateAccountAndLogin(createAccountRequest) require.NoError(t, err) - require.True(t, backupFecthed) + require.Equal(t, nameserver, testContext.backend.config.WakuV2Config.Nameserver) require.True(t, acc.HasAcceptedTerms) waitForLogin(c) - require.NoError(t, b.Logout()) - require.NoError(t, b.StopNode()) + require.NoError(t, testContext.backend.Logout()) + require.NoError(t, testContext.backend.StopNode()) + + testContext.backend.UpdateRootDataDir(testContext.config.DataDir) - accounts, err := b.GetAccounts() + accounts, err := testContext.backend.GetAccounts() require.NoError(t, err) require.Len(t, accounts, 1) @@ -890,289 +816,90 @@ func TestLoginAccount(t *testing.T) { loginAccountRequest := &requests.Login{ KeyUID: accounts[0].KeyUID, - Password: password, + Password: testPassword, WakuV2Nameserver: nameserver, } - err = b.LoginAccount(loginAccountRequest) + err = testContext.backend.LoginAccount(loginAccountRequest) require.NoError(t, err) waitForLogin(c) - - require.Equal(t, nameserver, b.config.WakuV2Config.Nameserver) + require.Equal(t, nameserver, testContext.backend.config.WakuV2Config.Nameserver) } func TestVerifyDatabasePassword(t *testing.T) { utils.Init() - b := NewGethStatusBackend(tt.MustCreateTestLogger()) - - chatKey, err := gethcrypto.GenerateKey() - require.NoError(t, err) - walletKey, err := gethcrypto.GenerateKey() - require.NoError(t, err) - keyUIDHex := sha256.Sum256(gethcrypto.FromECDSAPub(&chatKey.PublicKey)) - keyUID := types.EncodeHex(keyUIDHex[:]) - main := multiaccounts.Account{ - KeyUID: keyUID, - } - tmpdir := t.TempDir() - conf, err := params.NewNodeConfig(tmpdir, 1777) - require.NoError(t, err) - keyhex := hex.EncodeToString(gethcrypto.FromECDSA(chatKey)) - - require.NoError(t, b.AccountManager().InitKeystore(conf.KeyStoreDir)) - b.UpdateRootDataDir(conf.DataDir) - require.NoError(t, b.OpenAccounts()) - - address := crypto.PubkeyToAddress(walletKey.PublicKey) - - settings := testSettings - settings.KeyUID = keyUID - settings.Address = crypto.PubkeyToAddress(walletKey.PublicKey) - - chatPubKey := crypto.FromECDSAPub(&chatKey.PublicKey) - - require.NoError(t, b.SaveAccountAndStartNodeWithKey(main, "test-pass", settings, conf, []*accounts.Account{ - {Address: address, KeyUID: keyUID, Wallet: true}, - {Address: crypto.PubkeyToAddress(chatKey.PublicKey), KeyUID: keyUID, Chat: true, PublicKey: chatPubKey}}, keyhex)) - require.NoError(t, b.Logout()) - require.NoError(t, b.StopNode()) - - require.Error(t, b.VerifyDatabasePassword(main.KeyUID, "wrong-pass")) - require.NoError(t, b.VerifyDatabasePassword(main.KeyUID, "test-pass")) -} - -func TestDeleteMultiaccount(t *testing.T) { - backend := NewGethStatusBackend(tt.MustCreateTestLogger()) - - rootDataDir := t.TempDir() - - keyStoreDir := filepath.Join(rootDataDir, "keystore") - - backend.rootDataDir = rootDataDir - - err := backend.AccountManager().InitKeystore(keyStoreDir) - require.NoError(t, err) - - backend.AccountManager() - accs, err := backend.AccountManager(). - AccountsGenerator(). - GenerateAndDeriveAddresses(12, 1, "", []string{"m/44'/60'/0'/0"}) - require.NoError(t, err) - - generateAccount := accs[0] - accountInfo, err := backend.AccountManager(). - AccountsGenerator(). - StoreAccount(generateAccount.ID, "123123") - require.NoError(t, err) + testContext := setupTestContext(t, testPassword, false, false, false) - account := multiaccounts.Account{ - Name: "foo", - Timestamp: 1, - KeycardPairing: "pairing", - KeyUID: generateAccount.KeyUID, + request := &requests.CreateAccount{ + RootDataDir: testContext.config.DataDir, + Password: testPassword, + KdfIterations: 1, } - err = backend.ensureAppDBOpened(account, "123123") - require.NoError(t, err) - - s := settings.Settings{ - Address: types.HexToAddress(accountInfo.Address), - CurrentNetwork: "mainnet_rpc", - DappsAddress: types.HexToAddress(accountInfo.Address), - EIP1581Address: types.HexToAddress(accountInfo.Address), - InstallationID: "d3efcff6-cffa-560e-a547-21d3858cbc51", - KeyUID: account.KeyUID, - LatestDerivedPath: 0, - Name: "Jittery Cornflowerblue Kingbird", - Networks: &networks, - PhotoPath: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAIAAACRXR/mAAAAjklEQVR4nOzXwQmFMBAAUZXUYh32ZB32ZB02sxYQQSZGsod55/91WFgSS0RM+SyjA56ZRZhFmEWYRRT6h+M6G16zrxv6fdJpmUWYRbxsYr13dKfanpN0WmYRZhGzXz6AWYRZRIfbaX26fT9Jk07LLMIsosPt9I/dTDotswizCG+nhFmEWYRZhFnEHQAA///z1CFkYamgfQAAAABJRU5ErkJggg==", - PreviewPrivacy: false, - PublicKey: accountInfo.PublicKey, - SigningPhrase: "yurt joey vibe", - WalletRootAddress: types.HexToAddress(accountInfo.Address)} - - err = backend.saveAccountsAndSettings( - s, - ¶ms.NodeConfig{}, - nil) - require.Error(t, err) - require.True(t, err == accounts.ErrKeypairWithoutAccounts) - - err = backend.OpenAccounts() - require.NoError(t, err) - - err = backend.SaveAccount(account) - require.NoError(t, err) - - files, err := ioutil.ReadDir(rootDataDir) + _, err := testContext.backend.StartNodeWithChatKeyOrMnemonic( + request, + testContext.mnemonic, + nil, + false, + ) require.NoError(t, err) - require.NotEqual(t, 3, len(files)) - err = backend.DeleteMultiaccount(account.KeyUID, keyStoreDir) - require.NoError(t, err) + require.NoError(t, testContext.backend.Logout()) + require.NoError(t, testContext.backend.StopNode()) - files, err = ioutil.ReadDir(rootDataDir) - require.NoError(t, err) - require.Equal(t, 3, len(files)) + require.Error(t, testContext.backend.VerifyDatabasePassword(testContext.profileKeypair.KeyUID, "wrong-pass")) + require.NoError(t, testContext.backend.VerifyDatabasePassword(testContext.profileKeypair.KeyUID, testPassword)) } func TestConvertAccount(t *testing.T) { - const mnemonic = "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about" - const password = "111111" // represents password for a regular user - const keycardPassword = "222222" // represents password for a keycard user - const keycardUID = "1234" - const pathEIP1581Root = "m/43'/60'/1581'" - const pathEIP1581Chat = pathEIP1581Root + "/0'/0" - const pathWalletRoot = "m/44'/60'/0'/0" - const pathDefaultWalletAccount = pathWalletRoot + "/0" - const customWalletPath1 = pathWalletRoot + "/1" - const customWalletPath2 = pathWalletRoot + "/2" - var allGeneratedPaths []string - allGeneratedPaths = append(allGeneratedPaths, pathEIP1581Root, pathEIP1581Chat, pathWalletRoot, pathDefaultWalletAccount, customWalletPath1, customWalletPath2) - - var err error - - keystoreContainsFileForAccount := func(keyStoreDir string, hexAddress string) bool { - addrWithoutPrefix := strings.ToLower(hexAddress[2:]) - found := false - err = filepath.Walk(keyStoreDir, func(path string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if !fileInfo.IsDir() && strings.Contains(strings.ToUpper(path), strings.ToUpper(addrWithoutPrefix)) { - found = true - } - return nil - }) - return found - } - - rootDataDir := t.TempDir() + testContext := setupTestContext(t, testPassword, false, false, true) - keyStoreDir := filepath.Join(rootDataDir, "keystore") - - utils.Init() + request := &requests.CreateAccount{ + RootDataDir: testContext.config.DataDir, + Password: testPassword, + KdfIterations: 1, + } - backend, stop1, stop2, stopWallet, err := setupGethStatusBackend() - defer func() { - err := stop1() - if err != nil { - require.NoError(t, backend.StopNode()) - } - }() - defer func() { - err := stop2() - if err != nil { - require.NoError(t, backend.StopNode()) - } - }() - defer func() { - err := stopWallet() - if err != nil { - require.NoError(t, backend.StopNode()) - } - }() + _, err := testContext.backend.StartNodeWithChatKeyOrMnemonic( + request, + testContext.mnemonic, + nil, + false, + ) require.NoError(t, err) - backend.rootDataDir = rootDataDir - require.NoError(t, backend.AccountManager().InitKeystore(keyStoreDir)) - err = backend.OpenAccounts() + multiaccounts, err := testContext.backend.GetAccounts() require.NoError(t, err) + require.NotEmpty(t, multiaccounts[0].ColorHash) + serverMessenger := testContext.backend.Messenger() + require.NotNil(t, serverMessenger) - genAccInfo, err := backend.AccountManager().AccountsGenerator().ImportMnemonic(mnemonic, "") - assert.NoError(t, err) - - masterAddress := genAccInfo.Address - - accountInfo, err := backend.AccountManager().AccountsGenerator().StoreAccount(genAccInfo.ID, password) - assert.NoError(t, err) - - found := keystoreContainsFileForAccount(keyStoreDir, accountInfo.Address) - require.True(t, found) - - derivedAccounts, err := backend.AccountManager().AccountsGenerator().StoreDerivedAccounts(genAccInfo.ID, password, allGeneratedPaths) - assert.NoError(t, err) - - chatKey := derivedAccounts[pathEIP1581Chat].PrivateKey[2:] - chatAddress := derivedAccounts[pathEIP1581Chat].Address - found = keystoreContainsFileForAccount(keyStoreDir, chatAddress) - require.True(t, found) - - defaultSettings, err := defaultSettings(genAccInfo.KeyUID, genAccInfo.Address, derivedAccounts) + // Ensure all created accounts are in the keystore and can be loaded + masterAddress := types.HexToAddress(testContext.profileKeypair.DerivedFrom) + ok, err := testContext.backend.AccountsManager().VerifyAccountPassword(masterAddress, testPassword) require.NoError(t, err) - nodeConfig, err := DefaultNodeConfig(defaultSettings.InstallationID, genAccInfo.KeyUID, &requests.CreateAccount{ - LogLevel: defaultSettings.LogLevel, - }) - require.NoError(t, err) - nodeConfig.DataDir = rootDataDir - nodeConfig.KeyStoreDir = keyStoreDir - - profileKeypair := &accounts.Keypair{ - KeyUID: genAccInfo.KeyUID, - Name: "Profile Name", - Type: accounts.KeypairTypeProfile, - DerivedFrom: masterAddress, - } + require.True(t, ok) - profileKeypair.Accounts = append(profileKeypair.Accounts, &accounts.Account{ - Address: types.HexToAddress(chatAddress), - KeyUID: profileKeypair.KeyUID, - Type: accounts.AccountTypeGenerated, - PublicKey: types.Hex2Bytes(accountInfo.PublicKey), - Path: pathEIP1581Chat, - Wallet: false, - Chat: true, - Name: "GeneratedAccount", - }) - - for p, dAccInfo := range derivedAccounts { - found = keystoreContainsFileForAccount(keyStoreDir, dAccInfo.Address) + for _, acc := range testContext.profileKeypair.Accounts { + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(acc.Address, testPassword) require.NoError(t, err) - require.True(t, found) - - if p == pathDefaultWalletAccount || - p == customWalletPath1 || - p == customWalletPath2 { - wAcc := &accounts.Account{ - Address: types.HexToAddress(dAccInfo.Address), - KeyUID: genAccInfo.KeyUID, - Wallet: false, - Chat: false, - Type: accounts.AccountTypeGenerated, - Path: p, - Name: "derivacc" + p, - Hidden: false, - Removed: false, - } - if p == pathDefaultWalletAccount { - wAcc.Wallet = true - } - profileKeypair.Accounts = append(profileKeypair.Accounts, wAcc) - } - } - - account := multiaccounts.Account{ - Name: profileKeypair.Name, - Timestamp: 1, - KeyUID: profileKeypair.KeyUID, + require.True(t, ok) } - err = backend.ensureAppDBOpened(account, password) + // Ensure we're able to open the DB + err = testContext.backend.ensureAppDBOpened(*testContext.multiAcc, testPassword) require.NoError(t, err) - err = backend.StartNodeWithAccountAndInitialConfig(account, password, *defaultSettings, nodeConfig, profileKeypair.Accounts, nil) - require.NoError(t, err) - multiaccounts, err := backend.GetAccounts() + // db creation + db, err := accounts.NewDB(testContext.backend.appDB) require.NoError(t, err) - require.NotEmpty(t, multiaccounts[0].ColorHash) - serverMessenger := backend.Messenger() - require.NotNil(t, serverMessenger) - files, err := ioutil.ReadDir(rootDataDir) + // Check that there is no registered keycards + keycards, err := db.GetKeycardsWithSameKeyUID(testContext.profileKeypair.KeyUID) require.NoError(t, err) - require.NotEqual(t, 3, len(files)) + require.Equal(t, 0, len(keycards)) - keycardAccount := account + keycardAccount := *testContext.multiAcc keycardAccount.KeycardPairing = "pairing" keycardSettings := settings.Settings{ @@ -1181,103 +908,91 @@ func TestConvertAccount(t *testing.T) { KeycardPairing: "pairing", } - // Ensure we're able to open the DB - err = backend.ensureAppDBOpened(keycardAccount, keycardPassword) - require.NoError(t, err) - - // db creation - db, err := accounts.NewDB(backend.appDB) - require.NoError(t, err) - - // Check that there is no registered keycards - keycards, err := db.GetKeycardsWithSameKeyUID(genAccInfo.KeyUID) - require.NoError(t, err) - require.Equal(t, 0, len(keycards)) - // Converting to a keycard account - err = backend.ConvertToKeycardAccount(keycardAccount, keycardSettings, keycardUID, password, keycardPassword) + const keycardPassword = "222222" // represents password for a keycard user + err = testContext.backend.ConvertToKeycardAccount(keycardAccount, keycardSettings, testContext.profileKeypair.KeyUID, testPassword, keycardPassword) require.NoError(t, err) // Validating results of converting to a keycard account. // All keystore files for the account which is migrated need to be removed. - found = keystoreContainsFileForAccount(keyStoreDir, masterAddress) - require.False(t, found) + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(masterAddress, testPassword) + require.Error(t, err) + require.False(t, ok) - for _, dAccInfo := range derivedAccounts { - found = keystoreContainsFileForAccount(keyStoreDir, dAccInfo.Address) - require.False(t, found) + for _, acc := range testContext.profileKeypair.Accounts { + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(acc.Address, testPassword) + require.Error(t, err) + require.False(t, ok) } - require.NoError(t, backend.Logout()) - require.NoError(t, backend.StopNode()) + require.NoError(t, testContext.backend.Logout()) + require.NoError(t, testContext.backend.StopNode()) - require.NoError(t, backend.AccountManager().InitKeystore(keyStoreDir)) - require.NoError(t, backend.OpenAccounts()) + chatPrivKey := strings.TrimPrefix(testContext.chatPrivateKey, "0x") + require.NoError(t, testContext.backend.StartNodeWithKey(*testContext.multiAcc, keycardPassword, chatPrivKey, testContext.config)) - require.NoError(t, backend.StartNodeWithKey(account, keycardPassword, chatKey, nodeConfig)) defer func() { - assert.NoError(t, backend.Logout()) - assert.NoError(t, backend.StopNode()) + assert.NoError(t, testContext.backend.Logout()) + assert.NoError(t, testContext.backend.StopNode()) }() // Ensure we're able to open the DB - err = backend.ensureAppDBOpened(keycardAccount, keycardPassword) + err = testContext.backend.ensureAppDBOpened(keycardAccount, keycardPassword) require.NoError(t, err) // db creation after re-encryption - db1, err := accounts.NewDB(backend.appDB) + db1, err := accounts.NewDB(testContext.backend.appDB) require.NoError(t, err) // Check that there is a registered keycard - keycards, err = db1.GetKeycardsWithSameKeyUID(genAccInfo.KeyUID) + keycards, err = db1.GetKeycardsWithSameKeyUID(testContext.profileKeypair.KeyUID) require.NoError(t, err) require.Equal(t, 1, len(keycards)) // Converting to a regular account - err = backend.ConvertToRegularAccount(mnemonic, keycardPassword, password) + err = testContext.backend.ConvertToRegularAccount(testContext.mnemonic, keycardPassword, testPassword) require.NoError(t, err) // Validating results of converting to a regular account. // All keystore files for need to be created. - found = keystoreContainsFileForAccount(keyStoreDir, accountInfo.Address) - require.True(t, found) + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(masterAddress, testPassword) + require.NoError(t, err) + require.True(t, ok) - for _, dAccInfo := range derivedAccounts { - found = keystoreContainsFileForAccount(keyStoreDir, dAccInfo.Address) - require.True(t, found) + for _, acc := range testContext.profileKeypair.Accounts { + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(acc.Address, testPassword) + require.NoError(t, err) + require.True(t, ok) } - found = keystoreContainsFileForAccount(keyStoreDir, masterAddress) - require.True(t, found) - // Ensure we're able to open the DB - err = backend.ensureAppDBOpened(keycardAccount, password) + err = testContext.backend.ensureAppDBOpened(keycardAccount, testPassword) require.NoError(t, err) // db creation after re-encryption - db2, err := accounts.NewDB(backend.appDB) + db2, err := accounts.NewDB(testContext.backend.appDB) require.NoError(t, err) // Check that there is no registered keycards - keycards, err = db2.GetKeycardsWithSameKeyUID(genAccInfo.KeyUID) + keycards, err = db2.GetKeycardsWithSameKeyUID(testContext.profileKeypair.KeyUID) require.NoError(t, err) require.Equal(t, 0, len(keycards)) } func copyFile(srcFolder string, dstFolder string, fileName string, t *testing.T) { - data, err := ioutil.ReadFile(path.Join(srcFolder, fileName)) + data, err := os.ReadFile(path.Join(srcFolder, fileName)) if err != nil { t.Fail() } - err = ioutil.WriteFile(path.Join(dstFolder, fileName), data, 0600) + err = os.WriteFile(path.Join(dstFolder, fileName), data, 0600) if err != nil { t.Fail() } } func copyDir(srcFolder string, dstFolder string, t *testing.T) { - files, err := ioutil.ReadDir(srcFolder) + files, err := os.ReadDir(srcFolder) require.NoError(t, err) for _, file := range files { if !file.IsDir() { @@ -1292,34 +1007,32 @@ func copyDir(srcFolder string, dstFolder string, t *testing.T) { } } -func loginDesktopUser(t *testing.T, conf *params.NodeConfig) { +func loginDesktopUser(t *testing.T, conf *params.NodeConfig, keyUID string) { // The following passwords and DB used in this test unit are only // used to determine if login process works correctly after a migration // Expected account data: - keyUID := "0x7c46c8f6f059ab72d524f2a6d356904db30bb0392636172ab3929a6bd2220f84" // #nosec G101 username := "TestUser" passwd := "0xC888C9CE9E098D5864D3DED6EBCC140A12142263BACE3A23A36F9905F12BD64A" // #nosec G101 b := NewGethStatusBackend(tt.MustCreateTestLogger()) - require.NoError(t, b.AccountManager().InitKeystore(conf.KeyStoreDir)) b.UpdateRootDataDir(conf.DataDir) require.NoError(t, b.OpenAccounts()) - accounts, err := b.GetAccounts() + accs, err := b.GetAccounts() require.NoError(t, err) - require.Len(t, accounts, 1) - require.Equal(t, username, accounts[0].Name) - require.Equal(t, keyUID, accounts[0].KeyUID) + require.Len(t, accs, 1) + require.Equal(t, username, accs[0].Name) + require.Equal(t, keyUID, accs[0].KeyUID) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() - err := b.StartNodeWithAccount(accounts[0], passwd, conf, nil) + err := b.StartNodeWithAccount(accs[0], passwd, conf, nil) require.NoError(t, err) }() @@ -1333,92 +1046,97 @@ func loginDesktopUser(t *testing.T, conf *params.NodeConfig) { func TestLoginAndMigrationsStillWorkWithExistingDesktopUser(t *testing.T) { utils.Init() + keyUID := "0x7c46c8f6f059ab72d524f2a6d356904db30bb0392636172ab3929a6bd2220f84" // #nosec G101 + srcFolder := "../static/test-0.132.0-account/" tmpdir := t.TempDir() - copyDir(srcFolder, tmpdir, t) + keystoreDir := path.Join(tmpdir, "keystore", keyUID) + err := os.MkdirAll(keystoreDir, 0700) + require.NoError(t, err) + + srcKeystoreFolder := "../static/test-0.132.0-account/keystore/" + copyDir(srcKeystoreFolder, keystoreDir, t) + conf, err := params.NewNodeConfig(tmpdir, 1777) require.NoError(t, err) - loginDesktopUser(t, conf) - loginDesktopUser(t, conf) // Login twice to catch weird errors that only appear after logout + loginDesktopUser(t, conf, keyUID) + loginDesktopUser(t, conf, keyUID) // Login twice to catch weird errors that only appear after logout } func TestChangeDatabasePassword(t *testing.T) { - oldPassword := "password" - newPassword := "newPassword" - - backend := NewGethStatusBackend(tt.MustCreateTestLogger()) - backend.UpdateRootDataDir(t.TempDir()) + utils.Init() - // Setup keystore to test decryption of it - keyStoreDir := t.TempDir() - require.NoError(t, backend.accountManager.InitKeystore(keyStoreDir)) + testContext := setupTestContext(t, testPassword, true, true, false) - _, accountInfo, _, err := backend.accountManager.CreateAccount(oldPassword) + err := testContext.backend.StartNode(testContext.config) require.NoError(t, err) - account := multiaccounts.Account{ - Name: "TestAccount", - Timestamp: 1, - KeyUID: "0x7c46c8f6f059ab72d524f2a6d356904db30bb0392636172ab3929a6bd2220f84", - KDFIterations: 1, - } + defer func() { + require.NoError(t, testContext.backend.StopNode()) + }() - // Initialize accounts DB - err = backend.OpenAccounts() - require.NoError(t, err) - err = backend.SaveAccount(account) + masterAddress := types.HexToAddress(testContext.profileKeypair.DerivedFrom) + ok, err := testContext.backend.AccountsManager().VerifyAccountPassword(masterAddress, testPassword) require.NoError(t, err) + require.True(t, ok) - // Created DBs with old password - err = backend.ensureDBsOpened(account, oldPassword) + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(testContext.profileKeypair.Accounts[0].Address, testPassword) require.NoError(t, err) + require.True(t, ok) - // Change password - err = backend.ChangeDatabasePassword(account.KeyUID, oldPassword, newPassword) + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(testContext.profileKeypair.Accounts[1].Address, testPassword) require.NoError(t, err) + require.True(t, ok) - // Test that DBs can be opened with new password - appDbPath, err := backend.getAppDBPath(account.KeyUID) + db, err := testContext.backend.accountsDB() require.NoError(t, err) - appDb, err := sqlite.OpenDB(appDbPath, newPassword, account.KDFIterations) + + acc, err := db.GetAccountByAddress(testContext.profileKeypair.Accounts[0].Address) require.NoError(t, err) - appDb.Close() + require.Equal(t, testContext.profileKeypair.Accounts[0].Address, acc.Address) - walletDbPath, err := backend.getWalletDBPath(account.KeyUID) + acc, err = db.GetAccountByAddress(testContext.profileKeypair.Accounts[1].Address) require.NoError(t, err) - walletDb, err := sqlite.OpenDB(walletDbPath, newPassword, account.KDFIterations) + require.Equal(t, testContext.profileKeypair.Accounts[1].Address, acc.Address) + + // Change password + const newPassword = "newPassword" + err = testContext.backend.ChangeDatabasePassword(testContext.profileKeypair.KeyUID, testPassword, newPassword) require.NoError(t, err) - walletDb.Close() + + testContext.backend.UpdateRootDataDir(testContext.config.DataDir) // Test that keystore can be decrypted with the new password - acc, key, err := backend.accountManager.AddressToDecryptedAccount(accountInfo.WalletAddress, newPassword) + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(masterAddress, newPassword) require.NoError(t, err) - require.NotNil(t, acc) - require.NotNil(t, key) - require.Equal(t, acc.Address, key.Address) + require.True(t, ok) + + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(testContext.profileKeypair.Accounts[0].Address, newPassword) + require.NoError(t, err) + require.True(t, ok) + + ok, err = testContext.backend.AccountsManager().VerifyAccountPassword(testContext.profileKeypair.Accounts[1].Address, newPassword) + require.NoError(t, err) + require.True(t, ok) } func TestCreateWallet(t *testing.T) { utils.Init() - password := "some-password2" // nolint: goconst - tmpdir := t.TempDir() - b := NewGethStatusBackend(tt.MustCreateTestLogger()) - defer func() { - require.NoError(t, b.StopNode()) - }() + testContext := setupTestContext(t, testPassword, false, false, true) createAccountRequest := &requests.CreateAccount{ DisplayName: "some-display-name", CustomizationColor: "#ffffff", - Password: password, - RootDataDir: tmpdir, - LogFilePath: tmpdir + "/log", + Password: testPassword, + RootDataDir: testContext.config.DataDir, + LogFilePath: testContext.config.DataDir + "/log", } + c := make(chan interface{}, 10) signal.SetMobileSignalHandler(func(data []byte) { if strings.Contains(string(data), "node.login") { @@ -1427,9 +1145,9 @@ func TestCreateWallet(t *testing.T) { }) t.Cleanup(signal.ResetMobileSignalHandler) - account, err := b.CreateAccountAndLogin(createAccountRequest) + account, err := testContext.backend.CreateAccountAndLogin(createAccountRequest) require.NoError(t, err) - statusNode := b.statusNode + statusNode := testContext.backend.statusNode require.NotNil(t, statusNode) walletService := statusNode.WalletService() @@ -1438,16 +1156,16 @@ func TestCreateWallet(t *testing.T) { paths := []string{"m/44'/60'/0'/0/1"} - db, err := accounts.NewDB(b.appDB) + db, err := accounts.NewDB(testContext.backend.appDB) require.NoError(t, err) - walletRootAddress, err := db.GetWalletRootAddress() + masterAddress, err := db.GetMasterAddress() require.NoError(t, err) mnemonic, err := db.Mnemonic() require.NoError(t, err) require.NotEmpty(t, mnemonic) - derivedAddress, err := walletAPI.GetDerivedAddresses(context.Background(), password, walletRootAddress.String(), paths) + derivedAddress, err := walletAPI.GetDerivedAddresses(context.Background(), testPassword, masterAddress.String(), paths) require.NoError(t, err) require.Len(t, derivedAddress, 1) @@ -1455,9 +1173,10 @@ func TestCreateWallet(t *testing.T) { require.NotNil(t, accountsService) accountsAPI := accountsService.AccountsAPI() - err = accountsAPI.AddAccount(context.Background(), password, &accounts.Account{ + err = accountsAPI.AddAccount(context.Background(), testPassword, &accsmanagementtypes.Account{ + Address: derivedAddress[0].Address, KeyUID: account.KeyUID, - Type: accounts.AccountTypeGenerated, + Type: accsmanagementtypes.AccountTypeGenerated, PublicKey: derivedAddress[0].PublicKey, Emoji: "some", ColorID: "so", @@ -1469,17 +1188,17 @@ func TestCreateWallet(t *testing.T) { func TestSetFleet(t *testing.T) { utils.Init() - password := "some-password2" // nolint: goconst - tmpdir := t.TempDir() - b := NewGethStatusBackend(tt.MustCreateTestLogger()) + testContext := setupTestContext(t, testPassword, false, false, true) + createAccountRequest := &requests.CreateAccount{ DisplayName: "some-display-name", CustomizationColor: "#ffffff", - Password: password, - RootDataDir: tmpdir, - LogFilePath: tmpdir + "/log", + Password: testPassword, + RootDataDir: testContext.config.DataDir, + LogFilePath: testContext.config.DataDir + "/log", } + c := make(chan interface{}, 10) signal.SetMobileSignalHandler(func(data []byte) { if strings.Contains(string(data), "node.login") { @@ -1488,32 +1207,34 @@ func TestSetFleet(t *testing.T) { }) t.Cleanup(signal.ResetMobileSignalHandler) - newAccount, err := b.CreateAccountAndLogin(createAccountRequest) + newAccount, err := testContext.backend.CreateAccountAndLogin(createAccountRequest) require.NoError(t, err) - statusNode := b.statusNode + statusNode := testContext.backend.statusNode require.NotNil(t, statusNode) - savedSettings, err := b.GetSettings() + savedSettings, err := testContext.backend.GetSettings() require.NoError(t, err) require.Empty(t, savedSettings.Fleet) - accountsDB, err := b.accountsDB() + accountsDB, err := testContext.backend.accountsDB() require.NoError(t, err) err = accountsDB.SaveSettingField(settings.Fleet, params.FleetStatusProd) require.NoError(t, err) - savedSettings, err = b.GetSettings() + savedSettings, err = testContext.backend.GetSettings() require.NoError(t, err) require.NotEmpty(t, savedSettings.Fleet) require.Equal(t, params.FleetStatusProd, *savedSettings.Fleet) - require.NoError(t, b.Logout()) + require.NoError(t, testContext.backend.Logout()) + + testContext.backend.UpdateRootDataDir(testContext.config.DataDir) loginAccountRequest := &requests.Login{ KeyUID: newAccount.KeyUID, - Password: password, + Password: testPassword, } - require.NoError(t, b.LoginAccount(loginAccountRequest)) + require.NoError(t, testContext.backend.LoginAccount(loginAccountRequest)) select { case <-c: break @@ -1521,9 +1242,9 @@ func TestSetFleet(t *testing.T) { t.FailNow() } // Check is using the right fleet - require.Equal(t, b.config.ClusterConfig.WakuNodes, params.DefaultWakuNodes(params.FleetStatusProd)) + require.Equal(t, testContext.backend.config.ClusterConfig.WakuNodes, params.DefaultWakuNodes(params.FleetStatusProd)) - require.NoError(t, b.Logout()) + require.NoError(t, testContext.backend.Logout()) } func fakeToken() security.SensitiveString { @@ -1532,8 +1253,9 @@ func fakeToken() security.SensitiveString { func TestWalletConfigOnLoginAccount(t *testing.T) { utils.Init() - password := "some-password2" - tmpdir := t.TempDir() + + testContext := setupTestContext(t, testPassword, false, false, true) + poktToken := fakeToken() infuraToken := fakeToken() alchemyEthereumMainnetToken := fakeToken() @@ -1547,13 +1269,12 @@ func TestWalletConfigOnLoginAccount(t *testing.T) { raribleMainnetAPIKey := fakeToken() raribleTestnetAPIKey := fakeToken() - b := NewGethStatusBackend(tt.MustCreateTestLogger()) createAccountRequest := &requests.CreateAccount{ DisplayName: "some-display-name", CustomizationColor: "#ffffff", - Password: password, - RootDataDir: tmpdir, - LogFilePath: tmpdir + "/log", + Password: testPassword, + RootDataDir: testContext.config.DataDir, + LogFilePath: testContext.config.DataDir + "/log", } c := make(chan interface{}, 10) signal.SetMobileSignalHandler(func(data []byte) { @@ -1563,16 +1284,16 @@ func TestWalletConfigOnLoginAccount(t *testing.T) { }) t.Cleanup(signal.ResetMobileSignalHandler) - newAccount, err := b.CreateAccountAndLogin(createAccountRequest) + newAccount, err := testContext.backend.CreateAccountAndLogin(createAccountRequest) require.NoError(t, err) - statusNode := b.statusNode + statusNode := testContext.backend.statusNode require.NotNil(t, statusNode) - require.NoError(t, b.Logout()) + require.NoError(t, testContext.backend.Logout()) loginAccountRequest := &requests.Login{ KeyUID: newAccount.KeyUID, - Password: password, + Password: testPassword, WalletSecretsConfig: requests.WalletSecretsConfig{ PoktToken: poktToken, InfuraToken: infuraToken, @@ -1589,7 +1310,9 @@ func TestWalletConfigOnLoginAccount(t *testing.T) { }, } - require.NoError(t, b.LoginAccount(loginAccountRequest)) + testContext.backend.UpdateRootDataDir(testContext.config.DataDir) + + require.NoError(t, testContext.backend.LoginAccount(loginAccountRequest)) select { case <-c: break @@ -1597,19 +1320,20 @@ func TestWalletConfigOnLoginAccount(t *testing.T) { t.FailNow() } - require.Equal(t, b.config.WalletConfig.InfuraAPIKey, infuraToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.MainnetChainID], alchemyEthereumMainnetToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.SepoliaChainID], alchemyEthereumSepoliaToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.ArbitrumChainID], alchemyArbitrumMainnetToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.ArbitrumSepoliaChainID], alchemyArbitrumSepoliaToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.OptimismChainID], alchemyOptimismMainnetToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.OptimismSepoliaChainID], alchemyOptimismSepoliaToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.BaseChainID], alchemyBaseMainnetToken) - require.Equal(t, b.config.WalletConfig.AlchemyAPIKeys[common.BaseSepoliaChainID], alchemyBaseSepoliaToken) - require.Equal(t, b.config.WalletConfig.RaribleMainnetAPIKey, raribleMainnetAPIKey) - require.Equal(t, b.config.WalletConfig.RaribleTestnetAPIKey, raribleTestnetAPIKey) - - require.NoError(t, b.Logout()) + walletConfig := testContext.backend.config.WalletConfig + require.Equal(t, walletConfig.InfuraAPIKey, infuraToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.EthereumMainnet], alchemyEthereumMainnetToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.EthereumSepolia], alchemyEthereumSepoliaToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.ArbitrumMainnet], alchemyArbitrumMainnetToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.ArbitrumSepolia], alchemyArbitrumSepoliaToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.OptimismMainnet], alchemyOptimismMainnetToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.OptimismSepolia], alchemyOptimismSepoliaToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.BaseMainnet], alchemyBaseMainnetToken) + require.Equal(t, walletConfig.AlchemyAPIKeys[common.BaseSepolia], alchemyBaseSepoliaToken) + require.Equal(t, walletConfig.RaribleMainnetAPIKey, raribleMainnetAPIKey) + require.Equal(t, walletConfig.RaribleTestnetAPIKey, raribleTestnetAPIKey) + + require.NoError(t, testContext.backend.Logout()) } func TestTestnetEnabledSettingOnCreateAccount(t *testing.T) { @@ -1666,8 +1390,7 @@ func TestRestoreAccountAndLogin(t *testing.T) { // Test case 1: Valid restore account request restoreRequest := &requests.RestoreAccount{ - Mnemonic: "test test test test test test test test test test test test", - FetchBackup: false, + Mnemonic: "test test test test test test test test test test test test", CreateAccount: requests.CreateAccount{ DisplayName: "Account1", DeviceName: "StatusIM", @@ -1701,8 +1424,7 @@ func TestRestoreAccountAndLoginWithoutDisplayName(t *testing.T) { // Test case: Valid restore account request without DisplayName restoreRequest := &requests.RestoreAccount{ - Mnemonic: "test test test test test test test test test test test test", - FetchBackup: false, + Mnemonic: "test test test test test test test test test test test test", CreateAccount: requests.CreateAccount{ DeviceName: "StatusIM", Password: "password", @@ -1721,7 +1443,7 @@ func TestAcceptTerms(t *testing.T) { b := NewGethStatusBackend(tt.MustCreateTestLogger()) conf, err := params.NewNodeConfig(tmpdir, 1777) require.NoError(t, err) - require.NoError(t, b.AccountManager().InitKeystore(conf.KeyStoreDir)) + b.UpdateRootDataDir(conf.DataDir) require.NoError(t, b.OpenAccounts()) nameserver := "8.8.8.8" @@ -1829,8 +1551,7 @@ func TestRestoreKeycardAccountAndLogin(t *testing.T) { } exampleRequest := map[string]interface{}{ - "mnemonic": "", - "fetchBackup": true, + "mnemonic": "", "createAccountRequest": map[string]interface{}{ "rootDataDir": tmpdir, "kdfIterations": 256000, @@ -1886,7 +1607,6 @@ func TestRestoreKeycardAccountAndLogin(t *testing.T) { backend := NewGethStatusBackend(tt.MustCreateTestLogger()) require.NoError(t, err) - require.NoError(t, backend.AccountManager().InitKeystore(conf.KeyStoreDir)) backend.UpdateRootDataDir(conf.DataDir) require.NoError(t, backend.OpenAccounts()) diff --git a/api/defaults.go b/api/defaults.go index 55677d08c51..a9f2db30f6c 100644 --- a/api/defaults.go +++ b/api/defaults.go @@ -57,7 +57,6 @@ func defaultSettings(keyUID string, address string, derivedAddresses map[string] chatKeyString := derivedAddresses[pathDefaultChat].PublicKey s := &settings.Settings{} - s.BackupEnabled = true logLevel := "INFO" s.LogLevel = &logLevel s.ProfilePicturesShowTo = settings.ProfilePicturesShowToEveryone @@ -303,7 +302,6 @@ func DefaultNodeConfig(installationID, keyUID string, request *requests.CreateAc nodeConfig.LogDir = request.LogFilePath nodeConfig.LogLevel = DefaultLogLevel nodeConfig.DataDir = DefaultDataDir - nodeConfig.ProcessBackedupMessages = false nodeConfig.KeycardPairingDataFile = DefaultKeycardPairingDataFile if request.KeycardPairingDataFile != nil { nodeConfig.KeycardPairingDataFile = *request.KeycardPairingDataFile diff --git a/api/geth_backend.go b/api/geth_backend.go index ce9c8fd048d..fa11c23095c 100644 --- a/api/geth_backend.go +++ b/api/geth_backend.go @@ -7,8 +7,10 @@ import ( "encoding/json" "fmt" "math/big" + "net/http" "os" "path/filepath" + "runtime/debug" "strings" "sync" "time" @@ -25,21 +27,21 @@ import ( ethcrypto "github.com/ethereum/go-ethereum/crypto" signercore "github.com/ethereum/go-ethereum/signer/core/apitypes" - "github.com/status-im/status-go/account" - "github.com/status-im/status-go/account/generator" + accsmanagement "github.com/status-im/status-go/accounts-management" + accscommon "github.com/status-im/status-go/accounts-management/common" + "github.com/status-im/status-go/accounts-management/generator" + accsmanagementtypes "github.com/status-im/status-go/accounts-management/types" "github.com/status-im/status-go/appdatabase" "github.com/status-im/status-go/centralizedmetrics" centralizedmetricscommon "github.com/status-im/status-go/centralizedmetrics/common" gocommon "github.com/status-im/status-go/common" "github.com/status-im/status-go/common/dbsetup" "github.com/status-im/status-go/connection" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/images" - "github.com/status-im/status-go/internal/sentry" - "github.com/status-im/status-go/internal/version" + "github.com/status-im/status-go/internal/metrics" "github.com/status-im/status-go/logutils" - "github.com/status-im/status-go/metrics" "github.com/status-im/status-go/multiaccounts" "github.com/status-im/status-go/multiaccounts/accounts" multiacccommon "github.com/status-im/status-go/multiaccounts/common" @@ -47,7 +49,10 @@ import ( "github.com/status-im/status-go/node" "github.com/status-im/status-go/nodecfg" "github.com/status-im/status-go/params" + "github.com/status-im/status-go/pkg/sentry" + "github.com/status-im/status-go/pkg/version" "github.com/status-im/status-go/protocol" + "github.com/status-im/status-go/protocol/communities" identityutils "github.com/status-im/status-go/protocol/identity" "github.com/status-im/status-go/protocol/identity/colorhash" "github.com/status-im/status-go/protocol/requests" @@ -62,17 +67,10 @@ import ( "github.com/status-im/status-go/signal" "github.com/status-im/status-go/sqlite" "github.com/status-im/status-go/transactions" - wakutypes "github.com/status-im/status-go/waku/types" "github.com/status-im/status-go/walletdatabase" ) var ( - // ErrWhisperClearIdentitiesFailure clearing whisper identities has failed. - ErrWhisperClearIdentitiesFailure = errors.New("failed to clear whisper identities") - // ErrWhisperIdentityInjectionFailure injecting whisper identities has failed. - ErrWhisperIdentityInjectionFailure = errors.New("failed to inject identity into Whisper") - // ErrWakuIdentityInjectionFailure injecting whisper identities has failed. - ErrWakuIdentityInjectionFailure = errors.New("failed to inject identity into waku") // ErrUnsupportedRPCMethod is for methods not supported by the RPC interface ErrUnsupportedRPCMethod = errors.New("method is unsupported by RPC interface") // ErrRPCClientUnavailable is returned if an RPC client can't be retrieved. @@ -84,6 +82,13 @@ var ( var _ StatusBackend = (*GethStatusBackend)(nil) +type LoginParams struct { + ChatAddress types.Address `json:"chatAddress"` + Password string `json:"password"` + MainAccount types.Address `json:"mainAccount"` // TODO: remove this field + MultiAccount *multiaccounts.Account `json:"multiAccount"` +} + // GethStatusBackend implements the Status.im service over go-ethereum type GethStatusBackend struct { mu sync.Mutex @@ -94,14 +99,13 @@ type GethStatusBackend struct { config *params.NodeConfig statusNode *node.StatusNode - personalAPI *personal.PublicAPI + signer communities.MessageSigner multiaccountsDB *multiaccounts.Database account *multiaccounts.Account - accountManager *account.GethManager + accountsManager *accsmanagement.AccountsManager transactor *transactions.Transactor connectionState connection.State appState AppState - selectedAccountKeyID string allowAllRPC bool // used only for tests, disables api method restrictions LocalPairingStateManager *statecontrol.ProcessStateManager centralizedMetrics *centralizedmetrics.MetricService @@ -119,13 +123,20 @@ func NewGethStatusBackend(logger *zap.Logger) *GethStatusBackend { logger: logger, preLoginLogConfig: logutils.NewPreLoginLogConfig(), } - backend.initialize() + if err := backend.initialize(); err != nil { + logger.Error("failed to initialize backend", zap.Error(err)) + panic(err) + } logger.Info("Status backend initialized", zap.String("backend geth version", version.Version()), zap.String("commit", version.GitCommit()), zap.String("IpfsGatewayURL", params.IpfsGatewayURL)) + if gocommon.IsMobilePlatform() { + debug.SetMemoryLimit(1024 * 1024 * 150) // 150MB + } + return backend } @@ -133,19 +144,26 @@ func (b *GethStatusBackend) PreLoginLog() *logutils.PreLoginLogConfig { return b.preLoginLogConfig } -func (b *GethStatusBackend) initialize() { - accountManager := account.NewGethManager(b.logger) +func (b *GethStatusBackend) initialize() (err error) { + accountsManager, err := accsmanagement.NewAccountsManager(b.logger) + if err != nil { + b.logger.Error("failed to create new *AccountsManager instance", zap.Error(err)) + return + } + transactor := transactions.NewTransactor() - personalAPI := personal.NewAPI() - statusNode := node.New(transactor, b.logger) + personalService := personal.New() + statusNode := node.New(transactor, accountsManager, b.logger) b.statusNode = statusNode - b.accountManager = accountManager + b.accountsManager = accountsManager b.transactor = transactor - b.personalAPI = personalAPI + b.signer = personalService b.statusNode.SetMultiaccountsDB(b.multiaccountsDB) b.LocalPairingStateManager = new(statecontrol.ProcessStateManager) b.LocalPairingStateManager.SetPairing(false) + + return } // StatusNode returns reference to node manager @@ -153,9 +171,9 @@ func (b *GethStatusBackend) StatusNode() *node.StatusNode { return b.statusNode } -// AccountManager returns reference to account manager -func (b *GethStatusBackend) AccountManager() *account.GethManager { - return b.accountManager +// AccountsManager returns reference to accounts manager +func (b *GethStatusBackend) AccountsManager() *accsmanagement.AccountsManager { + return b.accountsManager } // Transactor returns reference to a status transactor @@ -163,9 +181,8 @@ func (b *GethStatusBackend) Transactor() *transactions.Transactor { return b.transactor } -// SelectedAccountKeyID returns a Whisper key ID of the selected chat key pair. -func (b *GethStatusBackend) SelectedAccountKeyID() string { - return b.selectedAccountKeyID +func (b *GethStatusBackend) MessageSigner() communities.MessageSigner { + return b.signer } // IsNodeRunning confirm that node is running @@ -181,6 +198,10 @@ func (b *GethStatusBackend) StartNode(config *params.NodeConfig) error { signal.SendNodeCrashed(err) return err } + + // Set initial connection state + b.statusNode.ConnectionChanged(b.connectionState) + return nil } @@ -188,6 +209,7 @@ func (b *GethStatusBackend) UpdateRootDataDir(datadir string) { b.mu.Lock() defer b.mu.Unlock() b.rootDataDir = datadir + b.accountsManager.SetRootDataDir(datadir) } func (b *GethStatusBackend) GetMultiaccountDB() *multiaccounts.Database { @@ -280,7 +302,7 @@ func (b *GethStatusBackend) StartPrometheusMetricsServer(address string) error { if b.prometheusMetrics != nil { return nil } - b.prometheusMetrics = metrics.NewMetricsServer(address, nil) + b.prometheusMetrics = metrics.NewMetricsServer(address) go b.prometheusMetrics.Listen() return nil } @@ -297,6 +319,9 @@ func (b *GethStatusBackend) getAccountByKeyUID(keyUID string) (*multiaccounts.Ac } for _, acc := range as { if acc.KeyUID == keyUID { + for k, v := range acc.Images { + acc.Images[k].LocalURL = b.statusNode.HTTPServer().MakeAccountImageURL(acc.KeyUID, v.Name, v.Clock) + } return &acc, nil } } @@ -312,82 +337,6 @@ func (b *GethStatusBackend) SaveAccount(account multiaccounts.Account) error { return b.multiaccountsDB.SaveAccount(account) } -func (b *GethStatusBackend) DeleteMultiaccount(keyUID string, keyStoreDir string) error { - b.mu.Lock() - defer b.mu.Unlock() - if b.multiaccountsDB == nil { - return errors.New("accounts db wasn't initialized") - } - - err := b.multiaccountsDB.DeleteAccount(keyUID) - if err != nil { - return err - } - - appDbPath, err := b.getAppDBPath(keyUID) - if err != nil { - return err - } - - walletDbPath, err := b.getWalletDBPath(keyUID) - if err != nil { - return err - } - - dbFiles := []string{ - filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", keyUID)), - filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql-shm", keyUID)), - filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql-wal", keyUID)), - filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db", keyUID)), - filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-shm", keyUID)), - filepath.Join(b.rootDataDir, fmt.Sprintf("%s.db-wal", keyUID)), - appDbPath, - appDbPath + "-shm", - appDbPath + "-wal", - walletDbPath, - walletDbPath + "-shm", - walletDbPath + "-wal", - } - for _, path := range dbFiles { - if _, err := os.Stat(path); err == nil { - err = os.Remove(path) - if err != nil { - return err - } - } - } - - if b.account != nil && b.account.KeyUID == keyUID { - // reset active account - b.account = nil - } - - return os.RemoveAll(keyStoreDir) -} - -func (b *GethStatusBackend) DeleteImportedKey(address, password, keyStoreDir string) error { - b.mu.Lock() - defer b.mu.Unlock() - - err := filepath.Walk(keyStoreDir, func(path string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if strings.Contains(fileInfo.Name(), address) { - _, err := b.accountManager.VerifyAccountPassword(keyStoreDir, "0x"+address, password) - if err != nil { - b.logger.Error("failed to verify account", zap.String("account", gocommon.TruncateWithDot(address)), zap.Error(err)) - return err - } - - return os.Remove(path) - } - return nil - }) - - return err -} - func (b *GethStatusBackend) runDBFileMigrations(account multiaccounts.Account, password string) (string, error) { // Migrate file path to fix issue https://github.com/status-im/status-go/issues/2027 unsupportedPath := filepath.Join(b.rootDataDir, fmt.Sprintf("app-%x.sql", account.KeyUID)) @@ -459,7 +408,16 @@ func (b *GethStatusBackend) ensureAppDBOpened(account multiaccounts.Account, pas b.logger.Error("failed to initialize db", zap.Error(err)) return err } + b.statusNode.SetAppDB(b.appDB) + + accountsDB, err := accounts.NewDB(b.appDB) + if err != nil { + b.logger.Error("failed to create new *Database instance", zap.Error(err)) + return + } + b.accountsManager.SetPersistence(accountsDB) + return nil } @@ -511,12 +469,8 @@ func (b *GethStatusBackend) ensureWalletDBOpened(account multiaccounts.Account, } func (b *GethStatusBackend) SetupLogSettings() error { - // sync pre_login.log - if err := logutils.ZapLogger().Sync(); err != nil { - return errors.Wrap(err, "failed to sync logger") - } - logSettings := b.config.ProfileLogSettings() - return logutils.OverrideRootLoggerWithConfig(logSettings) + _ = logutils.ZapLogger().Sync() + return logutils.OverrideRootLoggerWithConfig(b.config.ProfileLogSettings()) } // Deprecated: Use StartNodeWithAccount instead. @@ -571,11 +525,11 @@ func (b *GethStatusBackend) updateAccountColorHashAndColorID(keyUID string, acco if err != nil { return nil, err } - publicKey := keypair.GetChatPublicKey() - if publicKey == nil { - return nil, errors.New("chat public key not found") + chatAcc := keypair.GetChatAccount() + if chatAcc == nil { + return nil, errors.New("chat account not found") } - if err = enrichMultiAccountByPublicKey(multiAccount, publicKey); err != nil { + if err = EnrichMultiAccountByPublicKey(multiAccount, chatAcc.PublicKey); err != nil { return nil, err } if err = b.multiaccountsDB.UpdateAccount(*multiAccount); err != nil { @@ -630,7 +584,6 @@ func (b *GethStatusBackend) workaroundToFixBadMigration(request *requests.Login) // check if we saved a empty node config because of node config migration failed if currentConf.NetworkID == 0 && - currentConf.KeyStoreDir == "" && currentConf.DataDir == "" && currentConf.NodeKey == "" { // check if exist old node config @@ -678,10 +631,7 @@ func (b *GethStatusBackend) overridePartialWithOldNodeConfig(conf *params.NodeCo conf.LogDir = oldNodeConf.LogDir conf.LogLevel = oldNodeConf.LogLevel conf.DataDir = oldNodeConf.DataDir - conf.KeyStoreDir = oldNodeConf.KeyStoreDir conf.NodeKey = oldNodeConf.NodeKey - conf.RegisterTopics = oldNodeConf.RegisterTopics - conf.RequireTopics = oldNodeConf.RequireTopics } func (b *GethStatusBackend) convertLoginRequestToAccountRequest(loginRequest *requests.Login) *requests.CreateAccount { @@ -692,7 +642,6 @@ func (b *GethStatusBackend) convertLoginRequestToAccountRequest(loginRequest *re createAccount.WakuV2LightClient = loginRequest.WakuV2LightClient createAccount.WakuV2EnableMissingMessageVerification = loginRequest.WakuV2EnableMissingMessageVerification createAccount.WakuV2EnableStoreConfirmationForMessagesSent = loginRequest.WakuV2EnableStoreConfirmationForMessagesSent - createAccount.TelemetryServerURL = loginRequest.TelemetryServerURL createAccount.VerifyTransactionURL = loginRequest.VerifyTransactionURL createAccount.VerifyENSURL = loginRequest.VerifyENSURL createAccount.VerifyTransactionChainID = loginRequest.VerifyTransactionChainID @@ -706,22 +655,22 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error { } if request.Mnemonic != "" { - info, err := b.generateAccountInfo(request.Mnemonic) + generatedAccount, generatedAccountInfo, err := b.generateAccount(request.Mnemonic) if err != nil { return errors.Wrap(err, "failed to generate account info") } - if info.KeyUID != request.KeyUID { + if generatedAccountInfo.KeyUID != request.KeyUID { return errors.New("mnemonic does not match this account") } - derivedAddresses, err := b.getDerivedAddresses(info.ID) + _, generatedDerivedAccountsInfo, err := b.generateDerivedAddresses(generatedAccount, paths) if err != nil { - return errors.Wrap(err, "failed to get derived addresses") + return errors.Wrap(err, "failed to derive children accounts") } - request.Password = derivedAddresses[pathEncryption].PublicKey - request.KeycardWhisperPrivateKey = derivedAddresses[pathDefaultChat].PrivateKey + request.Password = generatedDerivedAccountsInfo[accscommon.PathEIP1581Encryption].PublicKey + request.KeycardWhisperPrivateKey = generatedDerivedAccountsInfo[accscommon.PathEIP1581Chat].PrivateKey } acc := multiaccounts.Account{ @@ -737,6 +686,8 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error { } } + b.UpdateRootDataDir(b.rootDataDir) + err := b.ensureDBsOpened(acc, request.Password) if err != nil { return errors.Wrap(err, "failed to open database") @@ -791,6 +742,12 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error { } b.account = multiAccount + err = b.StartNode(b.config) + if err != nil { + b.logger.Info("failed to start node") + return errors.Wrap(err, "failed to start node") + } + chatAddr, err := accountsDB.GetChatAddress() if err != nil { return errors.Wrap(err, "failed to get chat address") @@ -799,47 +756,15 @@ func (b *GethStatusBackend) loginAccount(request *requests.Login) error { if err != nil { return errors.Wrap(err, "failed to get wallet address") } - watchAddrs, err := accountsDB.GetWalletAddresses() - if err != nil { - return errors.Wrap(err, "failed to get wallet addresses") - } - login := account.LoginParams{ - Password: request.Password, - ChatAddress: chatAddr, - WatchAddresses: watchAddrs, - MainAccount: walletAddr, + login := LoginParams{ + Password: request.Password, + ChatAddress: chatAddr, + MainAccount: walletAddr, } - err = b.StartNode(b.config) + err = b.SelectAccount(login, request.ChatPrivateKey()) if err != nil { - b.logger.Info("failed to start node") - return errors.Wrap(err, "failed to start node") - } - - if chatKey := request.ChatPrivateKey(); chatKey == nil { - err = b.SelectAccount(login) - if err != nil { - return errors.Wrap(err, "failed to select account") - } - } else { - // In case of keycard, we don't have a keystore, instead we have private key loaded from the keycard - if err := b.accountManager.SetChatAccount(chatKey); err != nil { - return errors.Wrap(err, "failed to set chat account") - } - _, err = b.accountManager.SelectedChatAccount() - if err != nil { - return errors.Wrap(err, "failed to get selected chat account") - } - - b.accountManager.SetAccountAddresses(walletAddr, watchAddrs...) - err = b.injectAccountsIntoServices() - if err != nil { - return errors.Wrap(err, "failed to inject accounts into services") - } - } - - if err = b.statusNode.StartLocalBackup(); err != nil { - return err + return errors.Wrap(err, "failed to select account") } err = b.multiaccountsDB.UpdateAccountTimestamp(acc.KeyUID, time.Now().Unix()) @@ -888,6 +813,8 @@ func (b *GethStatusBackend) UpdateNodeConfigFleet(acc multiaccounts.Account, pas // Deprecated: Use loginAccount instead func (b *GethStatusBackend) startNodeWithAccount(acc multiaccounts.Account, password string, inputNodeCfg *params.NodeConfig, chatKey *ecdsa.PrivateKey) error { + b.UpdateRootDataDir(b.rootDataDir) + err := b.ensureDBsOpened(acc, password) if err != nil { return err @@ -913,55 +840,29 @@ func (b *GethStatusBackend) startNodeWithAccount(acc multiaccounts.Account, pass b.account = &acc - chatAddr, err := accountsDB.GetChatAddress() - if err != nil { - return err - } - walletAddr, err := accountsDB.GetWalletAddress() + err = b.StartNode(b.config) if err != nil { + b.logger.Info("failed to start node", zap.Error(err)) return err } - watchAddrs, err := accountsDB.GetWalletAddresses() + + chatAddr, err := accountsDB.GetChatAddress() if err != nil { return err } - login := account.LoginParams{ - Password: password, - ChatAddress: chatAddr, - WatchAddresses: watchAddrs, - MainAccount: walletAddr, - } - - err = b.StartNode(b.config) + walletAddr, err := accountsDB.GetWalletAddress() if err != nil { - b.logger.Info("failed to start node", zap.Error(err)) return err } - if chatKey == nil { - // Load account from keystore - err = b.SelectAccount(login) - if err != nil { - return err - } - } else { - // In case of keycard, we don't have keystore, but we directly have the private key - if err := b.accountManager.SetChatAccount(chatKey); err != nil { - return err - } - _, err = b.accountManager.SelectedChatAccount() - if err != nil { - return err - } - - b.accountManager.SetAccountAddresses(walletAddr, watchAddrs...) - err = b.injectAccountsIntoServices() - if err != nil { - return err - } + login := LoginParams{ + Password: password, + ChatAddress: chatAddr, + MainAccount: walletAddr, } - if err = b.statusNode.StartLocalBackup(); err != nil { + err = b.SelectAccount(login, chatKey) + if err != nil { return err } @@ -984,12 +885,12 @@ func (b *GethStatusBackend) GetSettings() (*settings.Settings, error) { return nil, err } - settings, err := accountsDB.GetSettings() + s, err := accountsDB.GetSettings() if err != nil { return nil, err } - return &settings, nil + return &s, nil } func (b *GethStatusBackend) GetEnsUsernames() ([]*ens.UsernameDetail, error) { @@ -998,36 +899,6 @@ func (b *GethStatusBackend) GetEnsUsernames() ([]*ens.UsernameDetail, error) { return db.GetEnsUsernames(&removed) } -func (b *GethStatusBackend) MigrateKeyStoreDir(acc multiaccounts.Account, password, oldDir, newDir string) error { - err := b.ensureDBsOpened(acc, password) - if err != nil { - return err - } - - accountDB, err := accounts.NewDB(b.appDB) - if err != nil { - return err - } - accounts, err := accountDB.GetActiveAccounts() - if err != nil { - return err - } - settings, err := accountDB.GetSettings() - if err != nil { - return err - } - addresses := []string{settings.EIP1581Address.Hex(), settings.WalletRootAddress.Hex()} - for _, account := range accounts { - addresses = append(addresses, account.Address.Hex()) - } - err = b.accountManager.MigrateKeyStoreDir(oldDir, newDir, addresses) - if err != nil { - return err - } - - return nil -} - func (b *GethStatusBackend) Login(keyUID, password string) error { return b.startNodeWithAccount(multiaccounts.Account{KeyUID: keyUID}, password, nil, nil) } @@ -1050,11 +921,11 @@ func (b *GethStatusBackend) LoggedIn(keyUID string, err error) error { signal.SendLoggedIn(nil, nil, nil, err) return err } - settings, err := b.GetSettings() + s, err := b.GetSettings() if err != nil { return err } - account, err := b.getAccountByKeyUID(keyUID) + acc, err := b.getAccountByKeyUID(keyUID) if err != nil { return err } @@ -1070,7 +941,7 @@ func (b *GethStatusBackend) LoggedIn(keyUID string, err error) error { return err } } - signal.SendLoggedIn(account, settings, ensUsernamesJSON, nil) + signal.SendLoggedIn(acc, s, ensUsernamesJSON, nil) return nil } @@ -1118,25 +989,15 @@ func (b *GethStatusBackend) ImportUnencryptedDatabase(acc multiaccounts.Account, } func (b *GethStatusBackend) reEncryptKeyStoreDir(currentPassword string, newPassword string) error { - config := b.StatusNode().Config() - keyDir := "" - if config == nil { - keyDir = b.accountManager.Keydir - } else { - keyDir = config.KeyStoreDir - } - - if keyDir != "" { - err := b.accountManager.ReEncryptKeyStoreDir(keyDir, currentPassword, newPassword) - if err != nil { - return fmt.Errorf("ReEncryptKeyStoreDir error: %v", err) - } + err := b.accountsManager.ReEncryptKeyStoreDir(currentPassword, newPassword) + if err != nil { + return fmt.Errorf("ReEncryptKeyStoreDir error: %v", err) } return nil } func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password string, newPassword string) error { - account, err := b.multiaccountsDB.GetAccount(keyUID) + acc, err := b.multiaccountsDB.GetAccount(keyUID) if err != nil { return err } @@ -1151,19 +1012,32 @@ func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password strin return err } - isCurrentAccount := appDBPath == internalDbPath + // In order to overcome Mac OS symlink issue, we check if the internalDbPath contains the appDBPath. + // Cause on macOS, `/var` is actually a symlink to `/private/var`. + isCurrentAccount := strings.Contains(internalDbPath, appDBPath) restartNode := func() { if isCurrentAccount { + pass := password + if err == nil { + pass = newPassword + } + + err := b.StopNode() + if err != nil { + b.logger.Error("failed to stop node", zap.Error(err)) + return + } + + // TODO https://github.com/status-im/status-go/issues/3906 + // Fix restarting node, as it always fails but the error is ignored + // because UI calls Logout and Quit afterwards. It should not be UI-dependent + // and should be handled gracefully here if it makes sense to run dummy node after + // logout + err = b.startNodeWithAccount(*acc, pass, b.config, nil) if err != nil { - // TODO https://github.com/status-im/status-go/issues/3906 - // Fix restarting node, as it always fails but the error is ignored - // because UI calls Logout and Quit afterwards. It should not be UI-dependent - // and should be handled gracefully here if it makes sense to run dummy node after - // logout - _ = b.startNodeWithAccount(*account, password, nil, nil) - } else { - _ = b.startNodeWithAccount(*account, newPassword, nil, nil) + b.logger.Error("failed to start node", zap.Error(err)) + return } } } @@ -1179,16 +1053,16 @@ func (b *GethStatusBackend) ChangeDatabasePassword(keyUID string, password strin // First change app DB password, because it also reencrypts the keystore, // otherwise if we call changeWalletDbPassword first and logout, we will fail // to reencrypt the keystore - err = b.changeAppDBPassword(account, logout, password, newPassword) + err = b.changeAppDBPassword(acc, logout, password, newPassword) if err != nil { return err } // Already logged out but pass a param to decouple the logic for testing - err = b.changeWalletDBPassword(account, noLogout, password, newPassword) + err = b.changeWalletDBPassword(acc, noLogout, password, newPassword) if err != nil { // Revert the password to original - err2 := b.changeAppDBPassword(account, noLogout, newPassword, password) + err2 := b.changeAppDBPassword(acc, noLogout, newPassword, password) if err2 != nil { b.logger.Error("failed to revert app db password", zap.Error(err2)) } @@ -1344,7 +1218,7 @@ func (b *GethStatusBackend) ConvertToKeycardAccount(account multiaccounts.Accoun keypair, err := accountDB.GetKeypairByKeyUID(account.KeyUID) if err != nil { - if err == accounts.ErrDbKeypairNotFound { + if err == accsmanagementtypes.ErrDbKeypairNotFound { return errors.New("cannot convert an unknown keypair") } return err @@ -1375,353 +1249,137 @@ func (b *GethStatusBackend) ConvertToKeycardAccount(account multiaccounts.Accoun return err } - // This check is added due to mobile app cause it doesn't support a Keycard features as desktop app. - // We should remove the following line once mobile and desktop app align. - if len(keycardUID) > 0 { - displayName, err := accountDB.DisplayName() - if err != nil { - return err - } - - position, err := accountDB.GetPositionForNextNewKeycard() - if err != nil { - return err - } - - kc := accounts.Keycard{ - KeycardUID: keycardUID, - KeycardName: displayName, - KeycardLocked: false, - KeyUID: account.KeyUID, - Position: position, - } - - for _, acc := range keypair.Accounts { - kc.AccountsAddresses = append(kc.AccountsAddresses, acc.Address) - } - err = messenger.SaveOrUpdateKeycard(context.Background(), &kc) - if err != nil { - return err - } - } - - masterAddress, err := accountDB.GetMasterAddress() - if err != nil { - return err - } - - eip1581Address, err := accountDB.GetEIP1581Address() - if err != nil { - return err - } - - walletRootAddress, err := accountDB.GetWalletRootAddress() - if err != nil { - return err - } - - err = b.closeDBs() + displayName, err := accountDB.DisplayName() if err != nil { return err } - err = b.ChangeDatabasePassword(account.KeyUID, oldPassword, newPassword) - if err != nil { - return err + kc := accsmanagementtypes.Keycard{ + KeycardUID: keycardUID, + KeycardName: displayName, + KeycardLocked: false, + KeyUID: account.KeyUID, } - // We need to delete all accounts for the Keycard which is being added for _, acc := range keypair.Accounts { - err = b.accountManager.DeleteAccount(acc.Address) - if err != nil { - return err - } - } - - err = b.accountManager.DeleteAccount(masterAddress) - if err != nil { - return err + kc.AccountsAddresses = append(kc.AccountsAddresses, acc.Address) } - - err = b.accountManager.DeleteAccount(eip1581Address) + err = messenger.SaveOrUpdateKeycard(context.Background(), &kc, oldPassword) if err != nil { return err } - err = b.accountManager.DeleteAccount(walletRootAddress) + err = b.closeDBs() if err != nil { return err } - return nil + return b.ChangeDatabasePassword(account.KeyUID, oldPassword, newPassword) } -func (b *GethStatusBackend) RestoreAccountAndLogin(request *requests.RestoreAccount, opts ...params.Option) (*multiaccounts.Account, error) { - - if err := request.Validate(); err != nil { +// CreateAccountAndLogin creates a new account and logs in with it. +func (b *GethStatusBackend) CreateAccountAndLogin(request *requests.CreateAccount) (*multiaccounts.Account, error) { + validation := &requests.CreateAccountValidation{ + AllowEmptyDisplayName: true, + } + if err := request.Validate(validation); err != nil { return nil, err } - response, err := b.generateOrImportAccount(request.Mnemonic, 0, request.FetchBackup, &request.CreateAccount, opts...) + mnemonic, err := accscommon.CreateRandomMnemonicWithDefaultLength() if err != nil { return nil, err } - err = b.StartNodeWithAccountAndInitialConfig( - *response.account, - request.Password, - *response.settings, - response.nodeConfig, - response.subAccounts, - response.chatPrivateKey, + return b.StartNodeWithChatKeyOrMnemonic( + request, + mnemonic, + nil, + false, ) +} - if err != nil { - b.logger.Error("start node", zap.Error(err)) +// RestoreAccountAndLogin restores an account and logs in with it. +func (b *GethStatusBackend) RestoreAccountAndLogin(request *requests.RestoreAccount) (*multiaccounts.Account, error) { + if err := request.Validate(false); err != nil { return nil, err } - return response.account, nil + return b.StartNodeWithChatKeyOrMnemonic( + &request.CreateAccount, + request.Mnemonic, + nil, + true, + ) } func (b *GethStatusBackend) RestoreKeycardAccountAndLogin(request *requests.RestoreAccount) (*multiaccounts.Account, error) { - if err := request.Validate(); err != nil { - return nil, err - } - - keyStoreDir, err := b.InitKeyStoreDirWithAccount(request.RootDataDir, request.Keycard.KeyUID) - if err != nil { - return nil, err - } - - derivedAddresses := map[string]generator.AccountInfo{ - pathDefaultChat: { - Address: request.Keycard.WhisperAddress, - PublicKey: request.Keycard.WhisperPublicKey, - PrivateKey: request.Keycard.WhisperPrivateKey, - }, - pathWalletRoot: { - Address: request.Keycard.WalletRootAddress, - }, - pathDefaultWallet: { - Address: request.Keycard.WalletAddress, - PublicKey: request.Keycard.WalletPublicKey, - }, - pathEIP1581: { - Address: request.Keycard.Eip1581Address, - }, - pathEncryption: { - PublicKey: request.Keycard.EncryptionPublicKey, - }, - } - - input := &prepareAccountInput{ - customizationColorClock: 0, - accountID: "", // empty for keycard - keyUID: request.Keycard.KeyUID, - address: request.Keycard.Address, - mnemonic: "", - restoringAccount: true, - derivedAddresses: derivedAddresses, - fetchBackup: request.FetchBackup, // WARNING: Ensure this value is correct - keyStoreDir: keyStoreDir, - } - - response, err := b.prepareNodeAccount(&request.CreateAccount, input) - if err != nil { + if err := request.Validate(true); err != nil { return nil, err } - err = b.StartNodeWithAccountAndInitialConfig( - *response.account, - request.Password, - *response.settings, - response.nodeConfig, - response.subAccounts, - response.chatPrivateKey, //request.WhisperPrivateKey, + return b.StartNodeWithChatKeyOrMnemonic( + &request.CreateAccount, + request.Mnemonic, + request.Keycard, + true, ) - - if err != nil { - b.logger.Error("start node", zap.Error(err)) - return nil, errors.Wrap(err, "failed to start node") - } - - return response.account, nil } func (b *GethStatusBackend) GetKeyUIDByMnemonic(mnemonic string) (string, error) { - accountGenerator := b.accountManager.AccountsGenerator() - - info, err := accountGenerator.ImportMnemonic(mnemonic, "") + genAccount, err := generator.CreateAccountFromMnemonic(mnemonic, "") if err != nil { return "", err } - return info.KeyUID, nil -} - -type prepareAccountInput struct { - customizationColorClock uint64 - accountID string - keyUID string - address string - mnemonic string - restoringAccount bool - derivedAddresses map[string]generator.AccountInfo - fetchBackup bool - keyStoreDir string - opts []params.Option -} + accInfo := genAccount.ToIdentifiedAccountInfo() -type accountBundle struct { - account *multiaccounts.Account - settings *settings.Settings - nodeConfig *params.NodeConfig - subAccounts []*accounts.Account - chatPrivateKey *ecdsa.PrivateKey + return accInfo.KeyUID, nil } -func (b *GethStatusBackend) generateOrImportAccount(mnemonic string, customizationColorClock uint64, fetchBackup bool, request *requests.CreateAccount, opts ...params.Option) (*accountBundle, error) { - info, err := b.generateAccountInfo(mnemonic) - if err != nil { - return nil, err +func (b *GethStatusBackend) generateAccount(mnemonic string) (genAcc *generator.Account, accInfo generator.GeneratedAccountInfo, err error) { + finalMnemonic := mnemonic + if mnemonic == "" { + finalMnemonic, err = accscommon.CreateRandomMnemonicWithDefaultLength() + if err != nil { + return + } } - keyStoreDir, err := b.InitKeyStoreDirWithAccount(request.RootDataDir, info.KeyUID) + genAcc, err = generator.CreateAccountFromMnemonic(finalMnemonic, "") if err != nil { - return nil, err + return } - derivedAddresses, err := b.getDerivedAddresses(info.ID) + accInfo = genAcc.ToGeneratedAccountInfo(finalMnemonic) + return +} + +func (b *GethStatusBackend) generateDerivedAddresses(genAcc *generator.Account, paths []string) (genDerivedAccounts map[string]*generator.Account, genDerivedAccountsInfo map[string]generator.AccountInfo, err error) { + genDerivedAccounts, err = generator.DeriveChildrenFromAccount(genAcc, paths) if err != nil { - return nil, err + return } - input := &prepareAccountInput{ - customizationColorClock: customizationColorClock, - accountID: info.ID, - keyUID: info.KeyUID, - address: info.Address, - mnemonic: info.Mnemonic, - restoringAccount: mnemonic != "", - derivedAddresses: derivedAddresses, - fetchBackup: fetchBackup, - keyStoreDir: keyStoreDir, - opts: opts, + genDerivedAccountsInfo = make(map[string]generator.AccountInfo, 0) + for path, acc := range genDerivedAccounts { + genDerivedAccountsInfo[path] = acc.ToAccountInfo() } - return b.prepareNodeAccount(request, input) + return } -func (b *GethStatusBackend) prepareNodeAccount(request *requests.CreateAccount, input *prepareAccountInput) (*accountBundle, error) { - var err error - response := &accountBundle{} - - if request.KeycardInstanceUID != "" { - request.Password = input.derivedAddresses[pathEncryption].PublicKey - } - - // NOTE: I intentionally left this condition separately and not an `else` branch. Technically it's an `else`, - // but the statements inside are not the opposite statement of the first statement. It's just kinda like this: - // - replace password when we're using keycard - // - store account when we're not using keycard - if request.KeycardInstanceUID == "" { - err = b.storeAccount(input.accountID, request.Password, paths) - if err != nil { - return nil, err - } - } - - response.account, err = b.buildAccount(request, input) - if err != nil { - return nil, errors.Wrap(err, "failed to build account") - } - - response.settings, err = b.prepareSettings(request, input) - if err != nil { - return nil, errors.Wrap(err, "failed to prepare settings") - } - - if response.account.Name == "" { - response.account.Name = response.settings.Name - } - - response.nodeConfig, err = b.prepareConfig(request, input, response.settings.InstallationID) - if err != nil { - return nil, errors.Wrap(err, "failed to prepare node config") - } - - response.subAccounts, err = b.prepareSubAccounts(request, input) - if err != nil { - return nil, errors.Wrap(err, "failed to prepare sub accounts") - } - - response, err = b.prepareForKeycard(request, input, response) - if err != nil { - return nil, errors.Wrap(err, "failed to prepare for keycard") - } - - return response, nil -} - -func (b *GethStatusBackend) InitKeyStoreDirWithAccount(rootDataDir, keyUID string) (string, error) { - b.UpdateRootDataDir(rootDataDir) - keyStoreRelativePath, keystoreAbsolutePath := DefaultKeystorePath(rootDataDir, keyUID) - // Initialize keystore dir with account - return keyStoreRelativePath, b.accountManager.InitKeystore(keystoreAbsolutePath) -} - -func (b *GethStatusBackend) generateAccountInfo(mnemonic string) (*generator.GeneratedAccountInfo, error) { - accountGenerator := b.accountManager.AccountsGenerator() - - var info generator.GeneratedAccountInfo - var err error - if mnemonic == "" { - // generate 1(n) account with default mnemonic length and no passphrase - generatedAccountInfos, err := accountGenerator.Generate(defaultMnemonicLength, 1, "") - info = generatedAccountInfos[0] - - if err != nil { - return nil, err - } - } else { - - info, err = accountGenerator.ImportMnemonic(mnemonic, "") - if err != nil { - return nil, err - } - } - - return &info, nil -} - -func (b *GethStatusBackend) storeAccount(id string, password string, paths []string) error { - accountGenerator := b.accountManager.AccountsGenerator() - - _, err := accountGenerator.StoreAccount(id, password) - if err != nil { - return err - } - - _, err = accountGenerator.StoreDerivedAccounts(id, password, paths) - if err != nil { - return err - } - - return nil -} - -func (b *GethStatusBackend) buildAccount(request *requests.CreateAccount, input *prepareAccountInput) (*multiaccounts.Account, error) { - err := b.OpenAccounts() - if err != nil { - return nil, err +func (b *GethStatusBackend) buildAccount(request *requests.CreateAccount, keyUID string, customizationColorClock uint64) (*multiaccounts.Account, error) { + err := b.OpenAccounts() + if err != nil { + return nil, err } acc := &multiaccounts.Account{ - KeyUID: input.keyUID, + KeyUID: keyUID, Name: request.DisplayName, CustomizationColor: multiacccommon.CustomizationColor(request.CustomizationColor), - CustomizationColorClock: input.customizationColorClock, + CustomizationColorClock: customizationColorClock, KDFIterations: request.KdfIterations, Timestamp: time.Now().Unix(), } @@ -1762,158 +1420,121 @@ func (b *GethStatusBackend) buildAccount(request *requests.CreateAccount, input return acc, nil } -func (b *GethStatusBackend) prepareSettings(request *requests.CreateAccount, input *prepareAccountInput) (*settings.Settings, error) { - settings, err := defaultSettings(input.keyUID, input.address, input.derivedAddresses) +func (b *GethStatusBackend) prepareSettings(request *requests.CreateAccount, mnemonic string, keyUID string, masterAddress string, + derivedAddresses map[string]generator.AccountInfo, restoreAccount bool) (*settings.Settings, error) { + s, err := defaultSettings(keyUID, masterAddress, derivedAddresses) if err != nil { return nil, err } - settings.DeviceName = request.DeviceName - settings.DisplayName = request.DisplayName - settings.PreviewPrivacy = request.PreviewPrivacy - settings.CurrentNetwork = request.CurrentNetwork - settings.TestNetworksEnabled = request.TestNetworksEnabled - settings.AutoRefreshTokensEnabled = request.AutoRefreshTokensEnabled - if !input.restoringAccount { - settings.Mnemonic = &input.mnemonic - // TODO(rasom): uncomment it as soon as address will be properly - // marked as shown on mobile client - //settings.MnemonicWasNotShown = true - } - - if !input.fetchBackup { - // This is a an account created from scratch, we can mark the BackupFetched as true - settings.BackupFetched = true + s.DeviceName = request.DeviceName + s.DisplayName = request.DisplayName + s.PreviewPrivacy = request.PreviewPrivacy + s.CurrentNetwork = request.CurrentNetwork + s.TestNetworksEnabled = request.TestNetworksEnabled + s.AutoRefreshTokensEnabled = request.AutoRefreshTokensEnabled + if !restoreAccount { + s.Mnemonic = &mnemonic + s.MnemonicWasNotShown = true } if request.WakuV2Fleet != "" { - settings.Fleet = &request.WakuV2Fleet + s.Fleet = &request.WakuV2Fleet } - return settings, nil + return s, nil } -func (b *GethStatusBackend) prepareConfig(request *requests.CreateAccount, input *prepareAccountInput, installationID string) (*params.NodeConfig, error) { - nodeConfig, err := DefaultNodeConfig(installationID, input.keyUID, request, input.opts...) +func (b *GethStatusBackend) prepareConfig(request *requests.CreateAccount, keyUID string, installationID string) (*params.NodeConfig, error) { + nodeConfig, err := DefaultNodeConfig(installationID, keyUID, request) if err != nil { return nil, err } - nodeConfig.ProcessBackedupMessages = input.fetchBackup - - // when we set nodeConfig.KeyStoreDir, value of nodeConfig.KeyStoreDir should not contain the rootDataDir - // loadNodeConfig will add rootDataDir to nodeConfig.KeyStoreDir - nodeConfig.KeyStoreDir = input.keyStoreDir return nodeConfig, nil } -func (b *GethStatusBackend) prepareSubAccounts(request *requests.CreateAccount, input *prepareAccountInput) ([]*accounts.Account, error) { - emoji, err := randomWalletEmoji() - if err != nil { - return nil, errors.Wrap(err, "failed to generate random emoji") +func (b *GethStatusBackend) prepareWalletAccount(request *requests.CreateAccount) *accsmanagementtypes.AccountCreationDetails { + return &accsmanagementtypes.AccountCreationDetails{ + Path: accscommon.PathDefaultWalletAccount, + Name: walletAccountDefaultName, + ColorID: request.CustomizationColor, } +} - walletDerivedAccount := input.derivedAddresses[pathDefaultWallet] - walletAccount := &accounts.Account{ - PublicKey: types.Hex2Bytes(walletDerivedAccount.PublicKey), - KeyUID: input.keyUID, - Address: types.HexToAddress(walletDerivedAccount.Address), - ColorID: multiacccommon.CustomizationColor(request.CustomizationColor), - Emoji: emoji, - Wallet: true, - Path: pathDefaultWallet, - Name: walletAccountDefaultName, - AddressWasNotShown: !input.restoringAccount, +func (b *GethStatusBackend) prepareKeypair(request *requests.CreateAccount, keyUID string, masterAddress string, + derivedAddresses map[string]generator.AccountInfo, restoreAccount bool) (keypair *accsmanagementtypes.Keypair, err error) { + // set up keypair + keypair = &accsmanagementtypes.Keypair{ + Name: request.DisplayName, + KeyUID: keyUID, + Type: accsmanagementtypes.KeypairTypeProfile, + DerivedFrom: masterAddress, + LastUsedDerivationIndex: 0, } - chatDerivedAccount := input.derivedAddresses[pathDefaultChat] - chatAccount := &accounts.Account{ + // add chat account + chatDerivedAccount := derivedAddresses[accscommon.PathEIP1581Chat] + keypair.Accounts = append(keypair.Accounts, &accsmanagementtypes.Account{ PublicKey: types.Hex2Bytes(chatDerivedAccount.PublicKey), - KeyUID: input.keyUID, + KeyUID: keypair.KeyUID, Address: types.HexToAddress(chatDerivedAccount.Address), - Name: request.DisplayName, Chat: true, - Path: pathDefaultChat, - } + Path: accscommon.PathEIP1581Chat, + Position: -1, // When creating a new account, the chat account should have position -1, cause it doesn't participate + Operable: accsmanagementtypes.AccountFullyOperable, + }) - return []*accounts.Account{walletAccount, chatAccount}, nil + // add wallet account + walletDerivedAccount := derivedAddresses[accscommon.PathDefaultWalletAccount] + keypair.Accounts = append(keypair.Accounts, &accsmanagementtypes.Account{ + PublicKey: types.Hex2Bytes(walletDerivedAccount.PublicKey), + KeyUID: keypair.KeyUID, + Address: types.HexToAddress(walletDerivedAccount.Address), + ColorID: multiacccommon.CustomizationColor(request.CustomizationColor), + Wallet: true, + Path: accscommon.PathDefaultWalletAccount, + Name: walletAccountDefaultName, + AddressWasNotShown: !restoreAccount, + Position: 0, // When creating a new account, the wallet account should have position 0, cause it's the default wallet account + Operable: accsmanagementtypes.AccountFullyOperable, + }) + + return } -func (b *GethStatusBackend) prepareForKeycard(request *requests.CreateAccount, input *prepareAccountInput, response *accountBundle) (*accountBundle, error) { +func (b *GethStatusBackend) prepareForKeycard(request *requests.CreateAccount, multiAccount *multiaccounts.Account, + settings *settings.Settings, nodeConfig *params.NodeConfig) error { if request.KeycardInstanceUID == "" { - return response, nil + return nil } if request.KeycardPairingKey != "" { // KeycardPairingKey is used only on mobile - response.settings.KeycardPairing = request.KeycardPairingKey - response.account.KeycardPairing = request.KeycardPairingKey + settings.KeycardPairing = request.KeycardPairingKey + multiAccount.KeycardPairing = request.KeycardPairingKey } else { // KeycardPairingDataFile is used only on desktop kp := wallet.NewKeycardPairings() - kp.SetKeycardPairingsFile(response.nodeConfig.KeycardPairingDataFile) + kp.SetKeycardPairingsFile(nodeConfig.KeycardPairingDataFile) pairings, err := kp.GetPairings() if err != nil { - return nil, errors.Wrap(err, "failed to get keycard pairings") + return errors.Wrap(err, "failed to get keycard pairings") } keycard, ok := pairings[request.KeycardInstanceUID] if !ok { - return nil, errors.New("keycard not found in pairings file") + return errors.New("keycard not found in pairings file") } - response.settings.KeycardPairing = keycard.Key - response.account.KeycardPairing = keycard.Key - } - - response.settings.KeycardInstanceUID = request.KeycardInstanceUID - response.settings.KeycardPairedOn = time.Now().Unix() - - privateKeyHex := strings.TrimPrefix(input.derivedAddresses[pathDefaultChat].PrivateKey, "0x") - var err error - response.chatPrivateKey, err = crypto.HexToECDSA(privateKeyHex) - if err != nil { - return nil, errors.Wrap(err, "failed to parse chat private key hex") + settings.KeycardPairing = keycard.Key + multiAccount.KeycardPairing = keycard.Key } - return response, nil -} - -func (b *GethStatusBackend) getDerivedAddresses(id string) (map[string]generator.AccountInfo, error) { - accountGenerator := b.accountManager.AccountsGenerator() - return accountGenerator.DeriveAddresses(id, paths) -} + settings.KeycardInstanceUID = request.KeycardInstanceUID + settings.KeycardPairedOn = time.Now().Unix() -// CreateAccountAndLogin creates a new account and logs in with it. -// NOTE: requests.CreateAccount is used for public, params.Option maybe used for internal usage. -func (b *GethStatusBackend) CreateAccountAndLogin(request *requests.CreateAccount, opts ...params.Option) (*multiaccounts.Account, error) { - validation := &requests.CreateAccountValidation{ - AllowEmptyDisplayName: true, - } - if err := request.Validate(validation); err != nil { - return nil, err - } - - response, err := b.generateOrImportAccount("", 1, false, request, opts...) - if err != nil { - return nil, err - } - - err = b.StartNodeWithAccountAndInitialConfig( - *response.account, - request.Password, - *response.settings, - response.nodeConfig, - response.subAccounts, - response.chatPrivateKey, - ) - - if err != nil { - b.logger.Error("start node", zap.Error(err)) - return nil, err - } - - return response.account, nil + return nil } func (b *GethStatusBackend) ConvertToRegularAccount(mnemonic string, currPassword string, newPassword string) error { @@ -1923,17 +1544,17 @@ func (b *GethStatusBackend) ConvertToRegularAccount(mnemonic string, currPasswor } mnemonicNoExtraSpaces := strings.Join(strings.Fields(mnemonic), " ") - accountInfo, err := b.accountManager.AccountsGenerator().ImportMnemonic(mnemonicNoExtraSpaces, "") + _, generatedAccountInfo, err := b.generateAccount(mnemonicNoExtraSpaces) if err != nil { return err } - kdfIterations, err := b.multiaccountsDB.GetAccountKDFIterationsNumber(accountInfo.KeyUID) + kdfIterations, err := b.multiaccountsDB.GetAccountKDFIterationsNumber(generatedAccountInfo.KeyUID) if err != nil { return err } - err = b.ensureDBsOpened(multiaccounts.Account{KeyUID: accountInfo.KeyUID, KDFIterations: kdfIterations}, currPassword) + err = b.ensureDBsOpened(multiaccounts.Account{KeyUID: generatedAccountInfo.KeyUID, KDFIterations: kdfIterations}, currPassword) if err != nil { return err } @@ -1949,32 +1570,25 @@ func (b *GethStatusBackend) ConvertToRegularAccount(mnemonic string, currPasswor } // We add these two paths, cause others will be added via `StoreAccount` function call - const pathWalletRoot = "m/44'/60'/0'/0" - const pathEIP1581 = "m/43'/60'/1581'" var paths []string - paths = append(paths, pathWalletRoot, pathEIP1581) + paths = append(paths, accscommon.PathWalletRoot, accscommon.PathEIP1581Root) for _, acc := range knownAccounts { - if accountInfo.KeyUID == acc.KeyUID { + if generatedAccountInfo.KeyUID == acc.KeyUID { paths = append(paths, acc.Path) } } - _, err = b.accountManager.AccountsGenerator().StoreAccount(accountInfo.ID, currPassword) - if err != nil { - return err - } - - _, err = b.accountManager.AccountsGenerator().StoreDerivedAccounts(accountInfo.ID, currPassword, paths) + _, _, err = b.accountsManager.StoreKeystoreFilesForMnemonic(mnemonicNoExtraSpaces, currPassword, paths) if err != nil { return err } - err = b.multiaccountsDB.UpdateAccountKeycardPairing(accountInfo.KeyUID, "") + err = b.multiaccountsDB.UpdateAccountKeycardPairing(generatedAccountInfo.KeyUID, "") if err != nil { return err } - err = messenger.DeleteAllKeycardsWithKeyUID(context.Background(), accountInfo.KeyUID) + err = messenger.DeleteAllKeycardsWithKeyUID(context.Background(), generatedAccountInfo.KeyUID) if err != nil { return err } @@ -2004,7 +1618,7 @@ func (b *GethStatusBackend) ConvertToRegularAccount(mnemonic string, currPasswor return err } - return b.ChangeDatabasePassword(accountInfo.KeyUID, currPassword, newPassword) + return b.ChangeDatabasePassword(generatedAccountInfo.KeyUID, currPassword, newPassword) } func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password string) error { @@ -2014,7 +1628,7 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin } if !b.appDBExists(keyUID) || !b.walletDBExists(keyUID) { - return errors.New("One or more databases not created") + return errors.New("one or more databases not created") } err = b.ensureDBsOpened(multiaccounts.Account{KeyUID: keyUID, KDFIterations: kdfIterations}, password) @@ -2030,36 +1644,8 @@ func (b *GethStatusBackend) VerifyDatabasePassword(keyUID string, password strin return nil } -func enrichMultiAccountBySubAccounts(account *multiaccounts.Account, subaccs []*accounts.Account) error { - if account.ColorHash != nil && account.ColorID != 0 { - return nil - } - - for i, acc := range subaccs { - subaccs[i].KeyUID = account.KeyUID - if acc.Chat { - pk := string(acc.PublicKey.Bytes()) - colorHash, err := colorhash.GenerateFor(pk) - if err != nil { - return err - } - account.ColorHash = colorHash - - colorID, err := identityutils.ToColorID(pk) - if err != nil { - return err - } - account.ColorID = colorID - - break - } - } - - return nil -} - -func enrichMultiAccountByPublicKey(account *multiaccounts.Account, publicKey types.HexBytes) error { - pk := string(publicKey.Bytes()) +func EnrichMultiAccountByPublicKey(account *multiaccounts.Account, chatPublicKey types.HexBytes) error { + pk := string(chatPublicKey.Bytes()) colorHash, err := colorhash.GenerateFor(pk) if err != nil { return err @@ -2075,67 +1661,186 @@ func enrichMultiAccountByPublicKey(account *multiaccounts.Account, publicKey typ return nil } -// Deprecated: Use CreateAccountAndLogin instead -func (b *GethStatusBackend) SaveAccountAndStartNodeWithKey( - account multiaccounts.Account, - password string, - settings settings.Settings, - nodecfg *params.NodeConfig, - subaccs []*accounts.Account, - keyHex string, -) error { - err := enrichMultiAccountBySubAccounts(&account, subaccs) +func (b *GethStatusBackend) StartNodeWithChatKeyOrMnemonic( + request *requests.CreateAccount, + mnemonic string, // empty mnemonic is used for keycard account, not empty for regular account + keycardData *requests.KeycardData, // nil for regular account, not nil for account with already set keycard + restoreAccount bool, +) (*multiaccounts.Account, error) { + // very important to update root data dir here + b.UpdateRootDataDir(request.RootDataDir) + + var ( + isKeycard = request.KeycardInstanceUID != "" + keyUID string + masterAddress string + chatPrivateKey *ecdsa.PrivateKey // set only for keycard account + chatPublicKey types.HexBytes + customizationColorClock uint64 // not sure if we need this customizationColorClock at all since the desktop app doesn't use it + derivedAddresses = map[string]generator.AccountInfo{ + accscommon.PathWalletRoot: {}, + accscommon.PathEIP1581Root: {}, + accscommon.PathEIP1581Chat: {}, + accscommon.PathDefaultWalletAccount: {}, + } + keypairToStoreDirectly *accsmanagementtypes.Keypair + ) + + if keycardData != nil { // means that the keycard is already set, details already on it + keyUID = keycardData.KeyUID + masterAddress = keycardData.Address + + derivedAddresses[accscommon.PathWalletRoot] = generator.AccountInfo{ + Address: keycardData.WalletRootAddress, + } + derivedAddresses[accscommon.PathEIP1581Root] = generator.AccountInfo{ + Address: keycardData.Eip1581Address, + } + derivedAddresses[accscommon.PathEIP1581Chat] = generator.AccountInfo{ + Address: keycardData.WhisperAddress, + PublicKey: keycardData.WhisperPublicKey, + PrivateKey: keycardData.WhisperPrivateKey, + } + derivedAddresses[accscommon.PathDefaultWalletAccount] = generator.AccountInfo{ + Address: keycardData.WalletAddress, + PublicKey: keycardData.WalletPublicKey, + } + derivedAddresses[accscommon.PathEIP1581Encryption] = generator.AccountInfo{ + PublicKey: keycardData.EncryptionPublicKey, + } + } else { + genMasterAcc, err := generator.CreateAccountFromMnemonic(mnemonic, "") + if err != nil { + return nil, err + } + + keyUID = genMasterAcc.KeyUID() + masterAddress = genMasterAcc.Address().Hex() + + if !restoreAccount { + customizationColorClock = 1 + } + + derivationPaths := []string{ + accscommon.PathWalletRoot, + accscommon.PathEIP1581Root, + accscommon.PathEIP1581Chat, + accscommon.PathDefaultWalletAccount, + accscommon.PathEIP1581Encryption, + } + _, derivedAddresses, err = b.generateDerivedAddresses(genMasterAcc, derivationPaths) + if err != nil { + return nil, err + } + } + + if isKeycard { + genChatAccount, err := generator.CreateAccountFromPrivateKey(derivedAddresses[accscommon.PathEIP1581Chat].PrivateKey) + if err != nil { + return nil, err + } + + chatPrivateKey = genChatAccount.PrivateKey() + chatPublicKey = types.Hex2Bytes(genChatAccount.PublicKeyHex()) + + request.Password = derivedAddresses[accscommon.PathEIP1581Encryption].PublicKey + } else { + chatPublicKey = types.Hex2Bytes(derivedAddresses[accscommon.PathEIP1581Chat].PublicKey) + } + + settings, err := b.prepareSettings(request, mnemonic, keyUID, masterAddress, derivedAddresses, restoreAccount) if err != nil { - return err + return nil, errors.Wrap(err, "failed to prepare settings") } - err = b.SaveAccount(account) + + multiAccount, err := b.buildAccount(request, keyUID, customizationColorClock) if err != nil { - return err + return nil, err } - err = b.ensureDBsOpened(account, password) + + if multiAccount.Name == "" { + multiAccount.Name = settings.Name + } + + err = EnrichMultiAccountByPublicKey(multiAccount, chatPublicKey) if err != nil { - return err + return nil, err } - err = b.saveAccountsAndSettings(settings, nodecfg, subaccs) + + nodeConfig, err := b.prepareConfig(request, keyUID, settings.InstallationID) if err != nil { - return err + return nil, errors.Wrap(err, "failed to prepare node config") + } + + err = b.ensureDBsOpened(*multiAccount, request.Password) + if err != nil { + return nil, err } - return b.StartNodeWithKey(account, password, keyHex, nodecfg) + + if isKeycard { + err = b.prepareForKeycard(request, multiAccount, settings, nodeConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to prepare for keycard") + } + + keypairToStoreDirectly, err = b.prepareKeypair(request, keyUID, masterAddress, derivedAddresses, restoreAccount) + if err != nil { + return nil, errors.Wrap(err, "failed to prepare keypair") + } + } else { + walletAccount := b.prepareWalletAccount(request) + _, err := b.accountsManager.CreateKeypairFromMnemonicAndStore(mnemonic, request.Password, + request.DisplayName, walletAccount, true, 0) + if err != nil { + return nil, err + } + } + + err = b.StartNodeWithAccountAndInitialConfig( + multiAccount, + request.Password, + *settings, + nodeConfig, + keypairToStoreDirectly, + chatPrivateKey, + ) + + return multiAccount, err } -// StartNodeWithAccountAndInitialConfig is used after account and config was generated. -// In current setup account name and config is generated on the client side. Once/if it will be generated on -// status-go side this flow can be simplified. -// TODO: Consider passing accountBundle here directly func (b *GethStatusBackend) StartNodeWithAccountAndInitialConfig( - account multiaccounts.Account, + multiAccount *multiaccounts.Account, password string, settings settings.Settings, nodecfg *params.NodeConfig, - subaccs []*accounts.Account, + keypair *accsmanagementtypes.Keypair, chatKey *ecdsa.PrivateKey, ) error { - err := enrichMultiAccountBySubAccounts(&account, subaccs) + err := b.ensureDBsOpened(*multiAccount, password) if err != nil { return err } - err = b.SaveAccount(account) + + err = b.SaveAccount(*multiAccount) if err != nil { return err } - err = b.ensureDBsOpened(account, password) + + err = b.saveKeypairAndSettings(settings, nodecfg, keypair) if err != nil { return err } - err = b.saveAccountsAndSettings(settings, nodecfg, subaccs) + + err = b.StartNodeWithAccount(*multiAccount, password, nodecfg, chatKey) if err != nil { + b.logger.Error("start node with account and initial config", zap.Error(err)) return err } - return b.StartNodeWithAccount(account, password, nodecfg, chatKey) + + return nil } -// TODO: change in `saveAccountsAndSettings` function param `subaccs []*accounts.Account` parameter to `profileKeypair *accounts.Keypair` parameter -func (b *GethStatusBackend) saveAccountsAndSettings(settings settings.Settings, nodecfg *params.NodeConfig, subaccs []*accounts.Account) error { +func (b *GethStatusBackend) saveKeypairAndSettings(settings settings.Settings, nodecfg *params.NodeConfig, keypair *accsmanagementtypes.Keypair) error { b.mu.Lock() defer b.mu.Unlock() accdb, err := accounts.NewDB(b.appDB) @@ -2147,37 +1852,11 @@ func (b *GethStatusBackend) saveAccountsAndSettings(settings settings.Settings, return err } - // In case of setting up new account either way (creating new, importing seed phrase, keycard account...) we should not - // back up any data after login, as it was the case before, that's the reason why we're setting last backup time to the time - // when an account was created. - now := time.Now().Unix() - err = accdb.SetLastBackup(uint64(now)) - if err != nil { - return err + if keypair != nil { + return accdb.SaveOrUpdateKeypair(keypair) } - keypair := &accounts.Keypair{ - KeyUID: settings.KeyUID, - Name: settings.DisplayName, - Type: accounts.KeypairTypeProfile, - DerivedFrom: settings.Address.String(), - LastUsedDerivationIndex: 0, - } - - // When creating a new account, the chat account should have position -1, cause it doesn't participate - // in the wallet view and default wallet account should be at position 0. - for _, acc := range subaccs { - if acc.Chat { - acc.Position = -1 - } - if acc.Wallet { - acc.Position = 0 - } - acc.Operable = accounts.AccountFullyOperable - keypair.Accounts = append(keypair.Accounts, acc) - } - - return accdb.SaveOrUpdateKeypair(keypair) + return nil } func (b *GethStatusBackend) loadNodeConfig(inputNodeCfg *params.NodeConfig) error { @@ -2201,15 +1880,8 @@ func (b *GethStatusBackend) loadNodeConfig(inputNodeCfg *params.NodeConfig) erro } } - // TODO: Consider removing the Enabled field from the config as WakuV1 has been removed. - conf.WakuV2Config.Enabled = true - // NodeConfig.Version should be taken from version.Version - // which is set at the compile time. - // What's cached is usually outdated so we overwrite it here. - conf.Version = version.Version() conf.RootDataDir = b.rootDataDir conf.DataDir = filepath.Join(b.rootDataDir, conf.DataDir) - conf.KeyStoreDir = filepath.Join(b.rootDataDir, conf.KeyStoreDir) if _, err = os.Stat(conf.RootDataDir); os.IsNotExist(err) { if err := os.MkdirAll(conf.RootDataDir, os.ModePerm); err != nil { @@ -2266,29 +1938,14 @@ func (b *GethStatusBackend) startNode(config *params.NodeConfig) (err error) { return err } - if b.accountManager.GetManager() == nil { - err = b.accountManager.InitKeystore(config.KeyStoreDir) - if err != nil { - return err - } - } - - manager := b.accountManager.GetManager() - if manager == nil { - return errors.New("ethereum accounts.Manager is nil") - } - - if err = b.statusNode.StartWithOptions(config, node.StartOptions{ - AccountsManager: manager, - }); err != nil { + if err = b.statusNode.Start(config); err != nil { return } - b.accountManager.SetRPCClient(b.statusNode.RPCClient(), rpc.DefaultCallTimeout) - signal.SendNodeStarted() b.transactor.SetNetworkID(config.NetworkID) b.transactor.SetRPC(b.statusNode.RPCClient(), rpc.DefaultCallTimeout) - b.personalAPI.SetRPC(b.statusNode.RPCClient(), rpc.DefaultCallTimeout) + + signal.SendNodeStarted() if err = b.registerHandlers(); err != nil { b.logger.Error("Handler registration failed", zap.Error(err)) @@ -2296,20 +1953,14 @@ func (b *GethStatusBackend) startNode(config *params.NodeConfig) (err error) { } b.logger.Info("Handlers registered") - // Handle a case when a node is stopped and resumed. - // If there is no account selected, an error is returned. - if _, err := b.accountManager.SelectedChatAccount(); err == nil { - if err := b.injectAccountsIntoServices(); err != nil { - return err - } - } else if err != account.ErrNoAccountSelected { - return err - } - if b.statusNode.WalletService() != nil { b.statusNode.WalletService().KeycardPairings().SetKeycardPairingsFile(config.KeycardPairingDataFile) } + if b.prometheusMetrics != nil { + b.prometheusMetrics.RegisterHandler("waku", b.wakuMetricsHandler()) + } + signal.SendNodeReady() return nil } @@ -2348,23 +1999,6 @@ func (b *GethStatusBackend) RestartNode() error { return b.startNode(b.config) } -// ResetChainData remove chain data from data directory. -// Node is stopped, and new node is started, with clean data directory. -func (b *GethStatusBackend) ResetChainData() error { - b.mu.Lock() - defer b.mu.Unlock() - - if err := b.stopNode(); err != nil { - return err - } - // config is cleaned when node is stopped - if err := b.statusNode.ResetChainData(b.config); err != nil { - return err - } - signal.SendChainDataRemoved() - return b.startNode(b.config) -} - // CallRPC executes public RPC requests on node's in-proc RPC server. func (b *GethStatusBackend) CallRPC(inputJSON string) (string, error) { client := b.statusNode.RPCClient() @@ -2425,37 +2059,37 @@ func (b *GethStatusBackend) SignMessage(rpcParams personal.SignParams) (types.He if err != nil { return types.HexBytes{}, err } - return b.personalAPI.Sign(rpcParams, verifiedAccount) + return b.signer.Sign(rpcParams, verifiedAccount) } // Recover calls the personalAPI to return address associated with the private // key that was used to calculate the signature in the message func (b *GethStatusBackend) Recover(rpcParams personal.RecoverParams) (types.Address, error) { - return b.personalAPI.Recover(rpcParams) + return b.signer.Recover(rpcParams) } // SignTypedData accepts data and password. Gets verified account and signs typed data. func (b *GethStatusBackend) SignTypedData(typed typeddata.TypedData, address string, password string) (types.HexBytes, error) { - account, err := b.getVerifiedWalletAccount(address, password) + acc, err := b.getVerifiedWalletAccount(address, password) if err != nil { return types.HexBytes{}, err } chain := new(big.Int).SetUint64(b.StatusNode().Config().NetworkID) - sig, err := typeddata.Sign(typed, account.AccountKey.PrivateKey, chain) + sig, err := typeddata.Sign(typed, acc.PrivateKey(), chain) if err != nil { return types.HexBytes{}, err } - return types.HexBytes(sig), err + return sig, err } // SignTypedDataV4 accepts data and password. Gets verified account and signs typed data. func (b *GethStatusBackend) SignTypedDataV4(typed signercore.TypedData, address string, password string) (types.HexBytes, error) { - account, err := b.getVerifiedWalletAccount(address, password) + acc, err := b.getVerifiedWalletAccount(address, password) if err != nil { return types.HexBytes{}, err } chain := new(big.Int).SetUint64(b.StatusNode().Config().NetworkID) - sig, err := typeddata.SignTypedDataV4(typed, account.AccountKey.PrivateKey, chain) + sig, err := typeddata.SignTypedDataV4(typed, acc.PrivateKey(), chain) if err != nil { return types.HexBytes{}, err } @@ -2482,72 +2116,8 @@ func (b *GethStatusBackend) HashTypedDataV4(typed signercore.TypedData) (types.H return types.Hash(hash), err } -func (b *GethStatusBackend) getVerifiedWalletAccount(address, password string) (*account.SelectedExtKey, error) { - config := b.StatusNode().Config() - db, err := accounts.NewDB(b.appDB) - if err != nil { - b.logger.Error("failed to create new *Database instance", zap.Error(err)) - return nil, err - } - exists, err := db.AddressExists(types.HexToAddress(address)) - if err != nil { - b.logger.Error("failed to query db for a given address", zap.String("address", gocommon.TruncateWithDot(address)), zap.Error(err)) - return nil, err - } - - if !exists { - b.logger.Error("failed to get a selected account", zap.Error(wallettypes.ErrInvalidTxSender)) - return nil, wallettypes.ErrAccountDoesntExist - } - - key, err := b.accountManager.VerifyAccountPassword(config.KeyStoreDir, address, password) - if _, ok := err.(*account.ErrCannotLocateKeyFile); ok { - key, err = b.generatePartialAccountKey(db, address, password) - if err != nil { - return nil, err - } - } - - if err != nil { - b.logger.Error("failed to verify account", zap.String("account", gocommon.TruncateWithDot(address)), zap.Error(err)) - return nil, err - } - - return &account.SelectedExtKey{ - Address: key.Address, - AccountKey: key, - }, nil -} - -func (b *GethStatusBackend) generatePartialAccountKey(db *accounts.Database, address string, password string) (*types.Key, error) { - dbPath, err := db.GetPath(types.HexToAddress(address)) - path := "m/" + dbPath[strings.LastIndex(dbPath, "/")+1:] - if err != nil { - b.logger.Error("failed to get path for given account address", zap.String("account", gocommon.TruncateWithDot(address)), zap.Error(err)) - return nil, err - } - - rootAddress, err := db.GetWalletRootAddress() - if err != nil { - return nil, err - } - info, err := b.accountManager.AccountsGenerator().LoadAccount(rootAddress.Hex(), password) - if err != nil { - return nil, err - } - masterID := info.ID - - accInfosMap, err := b.accountManager.AccountsGenerator().StoreDerivedAccounts(masterID, password, []string{path}) - if err != nil { - return nil, err - } - - _, key, err := b.accountManager.AddressToDecryptedAccount(accInfosMap[path].Address, password) - if err != nil { - return nil, err - } - - return key, nil +func (b *GethStatusBackend) getVerifiedWalletAccount(address, password string) (*generator.Account, error) { + return b.accountsManager.GetVerifiedWalletAccount(types.HexToAddress(address), password) } // registerHandlers attaches Status callback handlers to running node @@ -2564,7 +2134,7 @@ func (b *GethStatusBackend) registerHandlers() error { client.RegisterHandler( params.AccountsMethodName, func(context.Context, uint64, ...interface{}) (interface{}, error) { - return b.accountManager.Accounts() + return b.accountsManager.Accounts() }, ) @@ -2580,7 +2150,7 @@ func (b *GethStatusBackend) registerHandlers() error { return nil } -func unsupportedMethodHandler(ctx context.Context, chainID uint64, rpcParams ...interface{}) (interface{}, error) { +func unsupportedMethodHandler(_ context.Context, _ uint64, _ ...interface{}) (interface{}, error) { return nil, ErrUnsupportedRPCMethod } @@ -2678,7 +2248,7 @@ func (b *GethStatusBackend) Logout() error { return err } - b.AccountManager().Logout() + b.AccountsManager().Logout() b.account = nil if b.statusNode != nil { @@ -2697,7 +2267,9 @@ func (b *GethStatusBackend) Logout() error { } // re-initialize the node, at some point we should better manage the lifecycle - b.initialize() + if err = b.initialize(); err != nil { + return err + } err = b.statusNode.StartMediaServerWithoutDB() if err != nil { @@ -2712,16 +2284,12 @@ func (b *GethStatusBackend) Logout() error { // including in release builds, to help diagnose login issues. // related issue: https://github.com/status-im/status-mobile/issues/21501 func (b *GethStatusBackend) switchToPreLoginLog() error { - err := logutils.ZapLogger().Sync() - if err != nil { - return err - } + _ = logutils.ZapLogger().Sync() return logutils.OverrideRootLoggerWithConfig(b.preLoginLogConfig.ConvertToLogSettings()) } -// cleanupServices stops parts of services that doesn't managed by a node and removes injected data from services. +// cleanupServices stops parts of services that aren't managed by a node and removes injected data from services. func (b *GethStatusBackend) cleanupServices() error { - b.selectedAccountKeyID = "" if b.statusNode == nil { return nil } @@ -2762,13 +2330,11 @@ func (b *GethStatusBackend) closeWalletDB() error { // SelectAccount selects current wallet and chat accounts, by verifying that each address has corresponding account which can be decrypted // using provided password. Once verification is done, the decrypted chat key is injected into Whisper (as a single identity, // all previous identities are removed). -func (b *GethStatusBackend) SelectAccount(loginParams account.LoginParams) error { +func (b *GethStatusBackend) SelectAccount(loginParams LoginParams, privateKey *ecdsa.PrivateKey) error { b.mu.Lock() defer b.mu.Unlock() - b.AccountManager().RemoveOnboarding() - - err := b.accountManager.SelectAccount(loginParams) + err := b.accountsManager.SetChatAccount(loginParams.ChatAddress, loginParams.Password, privateKey) if err != nil { return err } @@ -2777,7 +2343,11 @@ func (b *GethStatusBackend) SelectAccount(loginParams account.LoginParams) error b.account = loginParams.MultiAccount } - if err := b.injectAccountsIntoServices(); err != nil { + if err := b.initProtocol(); err != nil { + return err + } + + if err = b.statusNode.StartLocalBackup(); err != nil { return err } @@ -2805,50 +2375,52 @@ func (b *GethStatusBackend) LocalPairingStarted() error { return accountDB.MnemonicWasShown() } -func (b *GethStatusBackend) injectAccountsIntoWakuService(w wakutypes.WakuKeyManager, st *ext.Service) error { - chatAccount, err := b.accountManager.SelectedChatAccount() +func (b *GethStatusBackend) initProtocol() error { + st := b.statusNode.WakuV2ExtService() + if st == nil { + return nil + } + chatAccount, err := b.accountsManager.SelectedChatAccount() if err != nil { return err } - - identity := chatAccount.AccountKey.PrivateKey - + identity := chatAccount.PrivateKey() acc, err := b.GetActiveAccount() if err != nil { return err } - - if err := w.DeleteKeyPairs(); err != nil { // err is not possible; method return value is incorrect - return err + params := ext.InitProtocolParams{ + Identity: identity, + AppDB: b.appDB, + WalletDB: b.walletDB, + HTTPServer: b.statusNode.HTTPServer(), + MultiAccountDB: b.multiaccountsDB, + Account: acc, + AccountsManager: b.accountsManager, + RPCClient: b.statusNode.RPCClient(), + WalletService: b.statusNode.WalletService(), + CommunityTokensService: b.statusNode.CommunityTokensService(), + AccountsPublisher: b.statusNode.AccountsPublisher(), + TimeSource: b.statusNode.TimeSource(), + MetricsEnabled: b.prometheusMetrics != nil, } - b.selectedAccountKeyID, err = w.AddKeyPair(identity) + err = st.InitProtocol(params) if err != nil { - return ErrWakuIdentityInjectionFailure + return err } - if st != nil { - if err := st.InitProtocol(b.statusNode.GethNode().Config().Name, identity, b.appDB, b.walletDB, - b.statusNode.HTTPServer(), b.multiaccountsDB, acc, b.accountManager, b.statusNode.RPCClient(), - b.statusNode.WalletService(), b.statusNode.CommunityTokensService(), b.statusNode.WakuV2Service(), - logutils.ZapLogger(), b.statusNode.AccountsFeed()); err != nil { - return err - } - // Set initial connection state - st.ConnectionChanged(b.connectionState) - - messenger := st.Messenger() - // Init public status api - b.statusNode.StatusPublicService().Init(messenger) - b.statusNode.AccountService().Init(messenger) - // Init chat service - accDB, err := accounts.NewDB(b.appDB) - if err != nil { - return err - } - b.statusNode.ChatService(accDB).Init(messenger) - b.statusNode.EnsService().Init(messenger.SyncEnsNamesWithDispatchMessage) - b.statusNode.CommunityTokensService().Init(messenger) + messenger := st.Messenger() + // Init public status api + b.statusNode.StatusPublicService().Init(messenger) + b.statusNode.AccountService().Init(messenger) + // Init chat service + accDB, err := accounts.NewDB(b.appDB) + if err != nil { + return err } + b.statusNode.ChatService(accDB).Init(messenger) + b.statusNode.EnsService().Init(messenger.SyncEnsNamesWithDispatchMessage) + b.statusNode.CommunityTokensService().Init(messenger) return nil } @@ -2869,19 +2441,6 @@ func (b *GethStatusBackend) KeyUID() string { return "" } -func (b *GethStatusBackend) injectAccountsIntoServices() error { - if b.statusNode.WakuV2Service() != nil { - return b.injectAccountsIntoWakuService(b.statusNode.WakuV2Service(), func() *ext.Service { - if b.statusNode.WakuV2ExtService() == nil { - return nil - } - return b.statusNode.WakuV2ExtService().Service - }()) - } - - return nil -} - // ExtractGroupMembershipSignatures extract signatures from tuples of content/signature func (b *GethStatusBackend) ExtractGroupMembershipSignatures(signaturePairs [][2]string) ([]string, error) { return crypto.ExtractSignatures(signaturePairs) @@ -2889,18 +2448,18 @@ func (b *GethStatusBackend) ExtractGroupMembershipSignatures(signaturePairs [][2 // SignGroupMembership signs a piece of data containing membership information func (b *GethStatusBackend) SignGroupMembership(content string) (string, error) { - selectedChatAccount, err := b.accountManager.SelectedChatAccount() + selectedChatAccount, err := b.accountsManager.SelectedChatAccount() if err != nil { return "", err } - return crypto.SignStringAsHex(content, selectedChatAccount.AccountKey.PrivateKey) + return crypto.SignStringAsHex(content, selectedChatAccount.PrivateKey()) } func (b *GethStatusBackend) Messenger() *protocol.Messenger { - node := b.StatusNode() - if node != nil { - accountService := node.AccountService() + statusNode := b.StatusNode() + if statusNode != nil { + accountService := statusNode.AccountService() if accountService != nil { return accountService.GetMessenger() } @@ -2915,12 +2474,12 @@ func (b *GethStatusBackend) SignHash(hexEncodedHash string) (string, error) { return "", fmt.Errorf("SignHash: could not unmarshal the input: %v", err) } - chatAccount, err := b.accountManager.SelectedChatAccount() + chatAccount, err := b.accountsManager.SelectedChatAccount() if err != nil { return "", fmt.Errorf("SignHash: could not select account: %v", err.Error()) } - signature, err := ethcrypto.Sign(hash, chatAccount.AccountKey.PrivateKey) + signature, err := ethcrypto.Sign(hash, chatAccount.PrivateKey()) if err != nil { return "", fmt.Errorf("SignHash: could not sign the hash: %v", err) } @@ -3044,3 +2603,26 @@ func (b *GethStatusBackend) SetPreLoginLogLevel(level string) error { } return logutils.OverrideRootLoggerWithConfig(b.preLoginLogConfig.ConvertToLogSettings()) } + +func (b *GethStatusBackend) wakuMetricsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if b.StatusNode() == nil { + b.logger.Error("failed to get waku metrics: StatusNode is nil") + return + } + + if b.StatusNode().WakuV2ExtService() == nil { + b.logger.Error("failed to get waku metrics: WakuV2ExtService is nil") + return + } + + wakuMetrics := b.StatusNode().WakuV2ExtService().Metrics() + if wakuMetrics != "" { + _, err := w.Write([]byte(wakuMetrics)) + + if err != nil { + b.logger.Error("failed to write waku metrics", zap.Error(err)) + } + } + }) +} diff --git a/appdatabase/migrations/sql/1755274768573_drop_waku_backup_setings.up.sql b/appdatabase/migrations/sql/1755274768573_drop_waku_backup_setings.up.sql new file mode 100644 index 00000000000..5a6f56ddfb9 --- /dev/null +++ b/appdatabase/migrations/sql/1755274768573_drop_waku_backup_setings.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE settings DROP COLUMN last_backup; +ALTER TABLE settings DROP COLUMN backup_enabled; \ No newline at end of file diff --git a/multiaccounts/accounts/database.go b/multiaccounts/accounts/database.go index 537851c2ad7..2025222cdd3 100644 --- a/multiaccounts/accounts/database.go +++ b/multiaccounts/accounts/database.go @@ -21,7 +21,6 @@ const ( statusChatPath = "m/43'/60'/1581'/0'/0" statusWalletRootPath = "m/44'/60'/0'/0/" zeroAddress = "0x0000000000000000000000000000000000000000" - SyncedFromBackup = "backup" // means a keypair is coming from backed up data ThirtyDaysInMilliseconds = 30 * 24 * 60 * 60 * 1000 maxNumOfGeneratedAddresses = uint64(100) diff --git a/multiaccounts/settings/columns.go b/multiaccounts/settings/columns.go index 0656b1f1b15..a2261da44f7 100644 --- a/multiaccounts/settings/columns.go +++ b/multiaccounts/settings/columns.go @@ -24,16 +24,6 @@ var ( dBColumnName: "auto_message_enabled", valueHandler: BoolHandler, } - BackupEnabled = SettingField{ - reactFieldName: "backup-enabled?", - dBColumnName: "backup_enabled", - valueHandler: BoolHandler, - } - BackupFetched = SettingField{ - reactFieldName: "backup-fetched?", - dBColumnName: "backup_fetched", - valueHandler: BoolHandler, - } BackupPath = SettingField{ reactFieldName: "backup-path", dBColumnName: "backup_path", @@ -159,10 +149,6 @@ var ( reactFieldName: "keycard-pairing", dBColumnName: "keycard_pairing", } - LastBackup = SettingField{ - reactFieldName: "last-backup", - dBColumnName: "last_backup", - } LastUpdated = SettingField{ reactFieldName: "last-updated", dBColumnName: "last_updated", @@ -550,8 +536,6 @@ var ( AnonMetricsShouldSend, Appearance, AutoMessageEnabled, - BackupEnabled, - BackupFetched, BackupPath, Bio, ChaosMode, @@ -576,7 +560,6 @@ var ( KeycardInstanceUID, KeycardPairedOn, KeycardPairing, - LastBackup, LastUpdated, LatestDerivedPath, LinkPreviewRequestEnabled, diff --git a/multiaccounts/settings/database.go b/multiaccounts/settings/database.go index 60d15047b34..64d672012a8 100644 --- a/multiaccounts/settings/database.go +++ b/multiaccounts/settings/database.go @@ -149,11 +149,10 @@ INSERT INTO settings ( test_networks_enabled, fleet, auto_refresh_tokens_enabled, - news_feed_last_fetched_timestamp, - backup_fetched + news_feed_last_fetched_timestamp ) VALUES ( ?,?,?,?,?,?,?,?,?,?,?,?,?,?, -?,?,?,?,?,?,?,?,?,'id',?,?,?,?,?,?,?,?,?,?,?,?,?)`, +?,?,?,?,?,?,?,?,?,'id',?,?,?,?,?,?,?,?,?,?,?,?)`, s.Address, s.Currency, s.CurrentNetwork, @@ -190,7 +189,6 @@ INSERT INTO settings ( s.AutoRefreshTokensEnabled, // Default the news feed last fetched timestamp to now time.Now().Unix(), - s.BackupFetched, ) if err != nil { return err @@ -404,7 +402,7 @@ func (db *Database) GetSettings() (Settings, error) { syncing_on_mobile_network, default_sync_period, use_mailservers, messages_from_contacts_only, usernames, appearance, profile_pictures_show_to, profile_pictures_visibility, wallet_root_address, wallet_set_up_passed, wallet_visible_tokens, waku_bloom_filter_mode, webview_allow_permission_requests, current_user_status, send_status_updates, gif_recents, - gif_favorites, opensea_enabled, last_backup, backup_enabled, telemetry_server_url, auto_message_enabled, gif_api_key, + gif_favorites, opensea_enabled, telemetry_server_url, auto_message_enabled, gif_api_key, test_networks_enabled, mutual_contact_enabled, profile_migration_needed, wallet_token_preferences_group_by_community, url_unfurling_mode, mnemonic_was_not_shown, wallet_show_community_asset_when_sending_tokens, wallet_display_assets_below_balance, wallet_display_assets_below_balance_threshold, wallet_collectible_preferences_group_by_collection, wallet_collectible_preferences_group_by_community, @@ -476,8 +474,6 @@ func (db *Database) GetSettings() (Settings, error) { &sqlite.JSONBlob{Data: &s.GifRecents}, &sqlite.JSONBlob{Data: &s.GifFavorites}, &s.OpenseaEnabled, - &s.LastBackup, - &s.BackupEnabled, &s.TelemetryServerURL, &s.AutoMessageEnabled, &s.GifAPIKey, @@ -627,14 +623,6 @@ func (db *Database) ShouldBroadcastUserStatus() (result bool, err error) { return result, err } -func (db *Database) BackupEnabled() (result bool, err error) { - err = db.makeSelectRow(BackupEnabled).Scan(&result) - if err == sql.ErrNoRows { - return true, nil - } - return result, err -} - func (db *Database) AutoMessageEnabled() (result bool, err error) { err = db.makeSelectRow(AutoMessageEnabled).Scan(&result) if err == sql.ErrNoRows { @@ -643,30 +631,6 @@ func (db *Database) AutoMessageEnabled() (result bool, err error) { return result, err } -func (db *Database) LastBackup() (result uint64, err error) { - err = db.makeSelectRow(LastBackup).Scan(&result) - if err == sql.ErrNoRows { - return 0, nil - } - return result, err -} - -func (db *Database) SetLastBackup(time uint64) error { - return db.SaveSettingField(LastBackup, time) -} - -func (db *Database) SetBackupFetched(fetched bool) error { - return db.SaveSettingField(BackupFetched, fetched) -} - -func (db *Database) BackupFetched() (result bool, err error) { - err = db.makeSelectRow(BackupFetched).Scan(&result) - if err == sql.ErrNoRows { - return true, nil - } - return result, err -} - func (db *Database) ENSName() (string, error) { return db.makeSelectString(PreferredName) } diff --git a/multiaccounts/settings/database_settings_manager.go b/multiaccounts/settings/database_settings_manager.go index 01b8139c02f..8365ca97d97 100644 --- a/multiaccounts/settings/database_settings_manager.go +++ b/multiaccounts/settings/database_settings_manager.go @@ -46,8 +46,6 @@ type DatabaseSettingsManager interface { SetSettingsNotifier(n Notifier) SetSettingLastSynced(setting SettingField, clock uint64) error - SetLastBackup(time uint64) error - SetBackupFetched(fetched bool) error SetPinnedMailservers(mailservers map[string]string) error SetUseMailservers(value bool) error SetTokenGroupByCommunity(value bool) error @@ -62,11 +60,8 @@ type DatabaseSettingsManager interface { CanUseMailservers() (result bool, err error) CanSyncOnMobileNetwork() (result bool, err error) ShouldBroadcastUserStatus() (result bool, err error) - BackupEnabled() (result bool, err error) BackupPath() (result string, err error) AutoMessageEnabled() (result bool, err error) - LastBackup() (result uint64, err error) - BackupFetched() (result bool, err error) ENSName() (string, error) DeviceName() (string, error) DisplayName() (string, error) diff --git a/multiaccounts/settings/database_test.go b/multiaccounts/settings/database_test.go index a09ccc9fc87..2b8ee2d258a 100644 --- a/multiaccounts/settings/database_test.go +++ b/multiaccounts/settings/database_test.go @@ -33,7 +33,6 @@ var ( DappsAddress: types.HexToAddress("0xD1300f99fDF7346986CbC766903245087394ecd0"), InstallationID: "d3efcff6-cffa-560e-a547-21d3858cbc51", KeyUID: "0x4e8129f3edfc004875be17bf468a784098a9f69b53c095be1f52deff286935ab", - BackupEnabled: true, LatestDerivedPath: 0, Name: "Jittery Cornflowerblue Kingbird", Networks: &networks, diff --git a/multiaccounts/settings/structs.go b/multiaccounts/settings/structs.go index 4b97f263da1..0472c5e1fc6 100644 --- a/multiaccounts/settings/structs.go +++ b/multiaccounts/settings/structs.go @@ -214,9 +214,6 @@ type Settings struct { OpenseaEnabled bool `json:"opensea-enabled?,omitempty"` TelemetryServerURL string `json:"telemetry-server-url,omitempty"` TelemetrySendPeriodMs int `json:"telemetry-send-period-ms,omitempty"` - LastBackup uint64 `json:"last-backup,omitempty"` - BackupEnabled bool `json:"backup-enabled?,omitempty"` - BackupFetched bool `json:"backup-fetched?,omitempty"` BackupPath string `json:"backup-path,omitempty"` AutoMessageEnabled bool `json:"auto-message-enabled?,omitempty"` GifAPIKey string `json:"gifs/api-key"` diff --git a/params/config.go b/params/config.go index 4984c05ecfa..3cff9c314d7 100644 --- a/params/config.go +++ b/params/config.go @@ -413,9 +413,6 @@ type NodeConfig struct { PushNotificationServerConfig PushNotificationServerConfig `json:"PushNotificationServerConfig"` OutputMessageCSVEnabled bool - - // ProcessBackedupMessages should be set to true when user follows recovery (using seed phrase or keycard) onboarding flow - ProcessBackedupMessages bool } // WalletConfig extra configuration for wallet.Service. diff --git a/protocol/activity_center.go b/protocol/activity_center.go index f059d831601..75a646fc919 100644 --- a/protocol/activity_center.go +++ b/protocol/activity_center.go @@ -42,10 +42,10 @@ const ( ActivityCenterNotificationTypeCommunityUnbanned ActivityCenterNotificationTypeNewInstallationReceived ActivityCenterNotificationTypeNewInstallationCreated - ActivityCenterNotificationTypeBackupSyncingFetching - ActivityCenterNotificationTypeBackupSyncingSuccess - ActivityCenterNotificationTypeBackupSyncingPartialFailure - ActivityCenterNotificationTypeBackupSyncingFailure + ActivityCenterNotificationTypeBackupSyncingFetching // Deprecated + ActivityCenterNotificationTypeBackupSyncingSuccess // Deprecated + ActivityCenterNotificationTypeBackupSyncingPartialFailure // Deprecated + ActivityCenterNotificationTypeBackupSyncingFailure // Deprecated ActivityCenterNotificationTypeNews ) diff --git a/protocol/common/feature_flags.go b/protocol/common/feature_flags.go index 004310af243..fe1c49a4e07 100644 --- a/protocol/common/feature_flags.go +++ b/protocol/common/feature_flags.go @@ -12,9 +12,6 @@ type FeatureFlags struct { // MailserverCycle indicates whether we should enable or not the mailserver cycle MailserverCycle bool - // DisableCheckingForBackup disables backup loop - DisableCheckingForBackup bool - // DisableAutoMessageLoop disables auto message loop DisableAutoMessageLoop bool diff --git a/protocol/communities_messenger_shared_member_address_test.go b/protocol/communities_messenger_shared_member_address_test.go index 93f48aa4114..3cf69d72cc0 100644 --- a/protocol/communities_messenger_shared_member_address_test.go +++ b/protocol/communities_messenger_shared_member_address_test.go @@ -1,24 +1,21 @@ package protocol import ( + "fmt" "testing" "time" "github.com/stretchr/testify/suite" - "go.uber.org/zap" gethcommon "github.com/ethereum/go-ethereum/common" hexutil "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/requests" - "github.com/status-im/status-go/protocol/tt" - - wakutypes "github.com/status-im/status-go/waku/types" ) func TestMessengerCommunitiesSharedMemberAddressSuite(t *testing.T) { @@ -26,17 +23,12 @@ func TestMessengerCommunitiesSharedMemberAddressSuite(t *testing.T) { } type MessengerCommunitiesSharedMemberAddressSuite struct { - suite.Suite + MessengerBaseTestSuite + owner *Messenger bob *Messenger alice *Messenger - ownerWaku wakutypes.Waku - bobWaku wakutypes.Waku - aliceWaku wakutypes.Waku - - logger *zap.Logger - mockedBalances map[uint64]map[gethcommon.Address]map[gethcommon.Address]*hexutil.Big // chainID, account, token, balance collectiblesServiceMock *CollectiblesServiceMock mockedCollectibles communities.CollectiblesByChain @@ -44,13 +36,7 @@ type MessengerCommunitiesSharedMemberAddressSuite struct { } func (s *MessengerCommunitiesSharedMemberAddressSuite) SetupTest() { - // Initialize with nil to avoid panics in TearDownTest - s.owner = nil - s.bob = nil - s.alice = nil - s.ownerWaku = nil - s.bobWaku = nil - s.aliceWaku = nil + s.MessengerBaseTestSuite.setupMessaging() communities.SetValidateInterval(300 * time.Millisecond) s.collectiblesServiceMock = &CollectiblesServiceMock{} @@ -61,19 +47,11 @@ func (s *MessengerCommunitiesSharedMemberAddressSuite) SetupTest() { s.resetMockedBalances() - s.logger = tt.MustCreateTestLogger() - - wakuNodes := CreateWakuV2Network(&s.Suite, s.logger, []string{"owner", "bob", "alice"}) - - s.ownerWaku = wakuNodes[0] - s.owner = s.newMessenger(ownerPassword, []string{ownerAddress}, s.ownerWaku, "owner", []Option{}) + s.owner = s.newMessenger(ownerPassword, []string{ownerAddress}, "owner", []Option{}) - s.bobWaku = wakuNodes[1] - s.bob = s.newMessenger(bobPassword, []string{bobAddress}, s.bobWaku, "bob", []Option{}) - s.bob.EnableBackedupMessagesProcessing() + s.bob = s.newMessenger(bobPassword, []string{bobAddress}, "bob", []Option{}) - s.aliceWaku = wakuNodes[2] - s.alice = s.newMessenger(alicePassword, []string{aliceAddress1, aliceAddress2}, s.aliceWaku, "alice", []Option{}) + s.alice = s.newMessenger(alicePassword, []string{aliceAddress1, aliceAddress2}, "alice", []Option{}) _, err := s.owner.Start() s.Require().NoError(err) @@ -87,27 +65,17 @@ func (s *MessengerCommunitiesSharedMemberAddressSuite) TearDownTest() { TearDownMessenger(&s.Suite, s.owner) TearDownMessenger(&s.Suite, s.bob) TearDownMessenger(&s.Suite, s.alice) - if s.ownerWaku != nil { - s.Require().NoError(s.ownerWaku.Stop()) - } - if s.bobWaku != nil { - s.Require().NoError(s.bobWaku.Stop()) - } - if s.aliceWaku != nil { - s.Require().NoError(s.aliceWaku.Stop()) - } - _ = s.logger.Sync() + s.MessengerBaseTestSuite.TearDownTest() } -func (s *MessengerCommunitiesSharedMemberAddressSuite) newMessenger(password string, walletAddresses []string, waku wakutypes.Waku, name string, extraOptions []Option) *Messenger { +func (s *MessengerCommunitiesSharedMemberAddressSuite) newMessenger(password string, walletAddresses []string, name string, extraOptions []Option) *Messenger { communityManagerOptions := []communities.ManagerOption{ communities.WithAllowForcingCommunityMembersReevaluation(true), } extraOptions = append(extraOptions, WithCommunityManagerOptions(communityManagerOptions)) - return newTestCommunitiesMessenger(&s.Suite, waku, testCommunitiesMessengerConfig{ + return newTestCommunitiesMessenger(&s.Suite, s.messagingEnv, testCommunitiesMessengerConfig{ testMessengerConfig: testMessengerConfig{ - logger: s.logger.Named(name), extraOptions: extraOptions, }, password: password, @@ -525,8 +493,8 @@ func (s *MessengerCommunitiesSharedMemberAddressSuite) TestResendSharedAddresses backupMessage, err := s.bob.backupCommunity(community, clock) s.Require().NoError(err) - err = s.bob.HandleBackup(s.bob.buildMessageState(), backupMessage, nil) - s.Require().NoError(err) + errs := s.bob.handleLocalBackupCommunities(s.bob.buildMessageState(), backupMessage.Communities) + s.Require().Len(errs, 0, fmt.Sprintf("expected no errors while handling backup communities. Errors: %v", errs)) // Owner will receive the request for addresses and send them back to Bob response, err := WaitOnMessengerResponse( @@ -608,8 +576,8 @@ func (s *MessengerCommunitiesSharedMemberAddressSuite) TestTokenMasterReceivesMe backupMessage, err := s.bob.backupCommunity(community, clock) s.Require().NoError(err) - err = s.bob.HandleBackup(s.bob.buildMessageState(), backupMessage, nil) - s.Require().NoError(err) + errs := s.bob.handleLocalBackupCommunities(s.bob.buildMessageState(), backupMessage.Communities) + s.Require().Len(errs, 0, fmt.Sprintf("expected no errors while handling backup communities. Errors: %v", errs)) // Owner will receive the request for addresses and send requests to join with revealed // addresses to token master diff --git a/protocol/communities_messenger_signers_test.go b/protocol/communities_messenger_signers_test.go index 6cc02bd5d2b..130ae43bd7c 100644 --- a/protocol/communities_messenger_signers_test.go +++ b/protocol/communities_messenger_signers_test.go @@ -609,7 +609,7 @@ func (s *MessengerCommunitiesSignersSuite) testSyncCommunity(mintOwnerToken bool // Create communities backup clock, _ := s.alice.getLastClockWithRelatedChat() - communitiesBackup, err := s.alice.backupCommunities(context.Background(), clock) + communitiesBackup, err := s.alice.backupCommunities(clock) s.Require().NoError(err) // Find wanted communities in the backup diff --git a/protocol/communities_messenger_token_permissions_test.go b/protocol/communities_messenger_token_permissions_test.go index 81339128d8a..44fe2c17c20 100644 --- a/protocol/communities_messenger_token_permissions_test.go +++ b/protocol/communities_messenger_token_permissions_test.go @@ -20,8 +20,8 @@ import ( gethcommon "github.com/ethereum/go-ethereum/common" hexutil "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/messaging" messagingtypes "github.com/status-im/status-go/messaging/types" "github.com/status-im/status-go/params" @@ -31,9 +31,6 @@ import ( "github.com/status-im/status-go/protocol/requests" "github.com/status-im/status-go/protocol/tt" "github.com/status-im/status-go/services/wallet/thirdparty" - "github.com/status-im/status-go/wakuv2" - - wakutypes "github.com/status-im/status-go/waku/types" ) const testChainID1 = 1 @@ -136,17 +133,12 @@ func TestMessengerCommunitiesTokenPermissionsSuite(t *testing.T) { } type MessengerCommunitiesTokenPermissionsSuite struct { - suite.Suite + MessengerBaseTestSuite + owner *Messenger bob *Messenger alice *Messenger - ownerWaku wakutypes.Waku - bobWaku wakutypes.Waku - aliceWaku wakutypes.Waku - - logger *zap.Logger - mockedBalances communities.BalancesByChain mockedCollectibles communities.CollectiblesByChain collectiblesServiceMock *CollectiblesServiceMock @@ -156,13 +148,7 @@ type MessengerCommunitiesTokenPermissionsSuite struct { } func (s *MessengerCommunitiesTokenPermissionsSuite) SetupTest() { - // Initialize with nil to avoid panics in TearDownTest - s.owner = nil - s.bob = nil - s.alice = nil - s.ownerWaku = nil - s.bobWaku = nil - s.aliceWaku = nil + s.MessengerBaseTestSuite.setupMessaging() s.accountsTestData = make(map[string][]string) s.accountsPasswords = make(map[string]string) @@ -173,19 +159,11 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) SetupTest() { } s.resetMockedBalances() - s.logger = tt.MustCreateTestLogger() - - wakuNodes := CreateWakuV2Network(&s.Suite, s.logger, []string{"owner", "bob", "alice"}) - - s.ownerWaku = wakuNodes[0] - s.owner = s.newMessenger(ownerPassword, []string{ownerAddress}, s.ownerWaku, "owner", []Option{}) + s.owner = s.newMessenger(ownerPassword, []string{ownerAddress}, "owner", []Option{}) - s.bobWaku = wakuNodes[1] - s.bob = s.newMessenger(bobPassword, []string{bobAddress}, s.bobWaku, "bob", []Option{}) - s.bob.EnableBackedupMessagesProcessing() + s.bob = s.newMessenger(bobPassword, []string{bobAddress}, "bob", []Option{}) - s.aliceWaku = wakuNodes[2] - s.alice = s.newMessenger(alicePassword, []string{aliceAddress1, aliceAddress2}, s.aliceWaku, "alice", []Option{}) + s.alice = s.newMessenger(alicePassword, []string{aliceAddress1, aliceAddress2}, "alice", []Option{}) _, err := s.owner.Start() s.Require().NoError(err) @@ -199,27 +177,17 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TearDownTest() { TearDownMessenger(&s.Suite, s.owner) TearDownMessenger(&s.Suite, s.bob) TearDownMessenger(&s.Suite, s.alice) - if s.ownerWaku != nil { - s.Require().NoError(s.ownerWaku.Stop()) - } - if s.bobWaku != nil { - s.Require().NoError(s.bobWaku.Stop()) - } - if s.aliceWaku != nil { - s.Require().NoError(s.aliceWaku.Stop()) - } - _ = s.logger.Sync() + s.MessengerBaseTestSuite.TearDownTest() } -func (s *MessengerCommunitiesTokenPermissionsSuite) newMessenger(password string, walletAddresses []string, waku wakutypes.Waku, name string, extraOptions []Option) *Messenger { +func (s *MessengerCommunitiesTokenPermissionsSuite) newMessenger(password string, walletAddresses []string, name string, extraOptions []Option) *Messenger { communityManagerOptions := []communities.ManagerOption{ communities.WithAllowForcingCommunityMembersReevaluation(true), } extraOptions = append(extraOptions, WithCommunityManagerOptions(communityManagerOptions)) - messenger := newTestCommunitiesMessenger(&s.Suite, waku, testCommunitiesMessengerConfig{ + messenger := newTestCommunitiesMessenger(&s.Suite, s.messagingEnv, testCommunitiesMessengerConfig{ testMessengerConfig: testMessengerConfig{ - logger: s.logger.Named(name), extraOptions: extraOptions, }, password: password, @@ -479,57 +447,8 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestRequestAccessWithENSToke } } -// NOTE(cammellos): Disabling for now as flaky, for some reason does not pass on CI, but passes locally func (s *MessengerCommunitiesTokenPermissionsSuite) TestBecomeMemberPermissions() { - s.T().Skip("flaky test") - - // Create a store node - // This is needed to fetch the messages after rejoining the community - var err error - - cfg := testWakuV2Config{ - logger: s.logger.Named("store-node-waku"), - enableStore: false, - clusterID: wakuv2.MainStatusShardCluster, - } - wakuStoreNode := NewTestWakuV2(&s.Suite, cfg) - - storeNodeListenAddresses, err := wakuStoreNode.ListenAddresses() - s.Require().NoError(err) - s.Require().LessOrEqual(1, len(storeNodeListenAddresses)) - - storeNodeAddress := storeNodeListenAddresses[0] - s.logger.Info("store node ready", zap.Stringer("address", storeNodeAddress)) - - // Create messengers - - wakuNodes := CreateWakuV2Network(&s.Suite, s.logger, []string{"owner", "bob"}) - s.ownerWaku = wakuNodes[0] - s.bobWaku = wakuNodes[1] - - options := []Option{ - WithTestStoreNode(&s.Suite, localMailserverID, storeNodeAddress, localFleet, s.collectiblesServiceMock), - } - - s.owner = s.newMessenger(ownerPassword, []string{ownerAddress}, s.ownerWaku, "owner", options) - s.Require().NoError(err) - - _, err = s.owner.Start() - s.Require().NoError(err) - - s.bob = s.newMessenger(bobPassword, []string{bobAddress}, s.bobWaku, "bob", options) - s.Require().NoError(err) - - _, err = s.bob.Start() - s.Require().NoError(err) - - // Force the owner to use the store node as relay peer - - err = s.owner.DialPeer(storeNodeAddress) - s.Require().NoError(err) - // Create a community - community, chat := s.createCommunity() // bob joins the community @@ -544,7 +463,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestBecomeMemberPermissions( // send message to the channel msg := s.sendChatMessage(s.owner, chat.ID, messages[0]) - s.logger.Debug("owner sent a message", + s.T().Log("owner sent a message", zap.String("messageText", msg.Text), zap.String("messageID", msg.ID), ) @@ -576,11 +495,11 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestBecomeMemberPermissions( CommunityID: community.ID(), Type: protobuf.CommunityTokenPermission_BECOME_MEMBER, TokenCriteria: []*protobuf.TokenCriteria{ - &protobuf.TokenCriteria{ + { Type: protobuf.CommunityTokenType_ERC20, ContractAddresses: map[uint64]string{testChainID1: "0x123"}, Symbol: "TEST", - Amount: "100", + AmountInWei: "100000000000000000000", Decimals: uint64(18), }, }, @@ -615,23 +534,6 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestBecomeMemberPermissions( ) s.Require().NoError(err) - // We are not member of the community anymore, so we need to refetch - // the data, since we would not be pulling it anymore - _, err = WaitOnMessengerResponse( - s.bob, - func(r *MessengerResponse) bool { - _, err := s.bob.FetchCommunity(&FetchCommunityRequest{WaitForResponse: true, TryDatabase: false, CommunityKey: community.IDString()}) - if err != nil { - return false - } - c, err := s.bob.communitiesManager.GetByID(community.ID()) - return err == nil && c != nil && len(c.TokenPermissions()) > 0 && !c.Joined() - }, - "no token permissions", - ) - - s.Require().NoError(err) - // bob tries to join, but he doesn't satisfy so the request isn't sent request := s.createRequestToJoinCommunity(community.ID(), s.bob) _, err = s.bob.RequestToJoinCommunity(request) @@ -662,7 +564,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestBecomeMemberPermissions( // send message to channel msg = s.sendChatMessage(s.owner, chat.ID, messages[2]) - s.logger.Debug("owner sent a message", + s.T().Log("owner sent a message", zap.String("messageText", msg.Text), zap.String("messageID", msg.ID), ) @@ -909,7 +811,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) testViewChannelPermissions(v waitOnBobToBeKickedFromChannel := waitOnCommunitiesEvent(s.owner, func(sub *communities.Subscription) bool { channel, ok := sub.Community.Chats()[chat.CommunityChatID()] - return ok && len(channel.Members) == 1 + return ok && len(channel.Members) == 0 }) waitOnChannelToBeRekeyedOnceBobIsKicked := s.waitOnKeyDistribution(func(sub *CommunityAndKeyActions) bool { action, ok := sub.keyActions.ChannelKeysActions[chat.CommunityChatID()] @@ -936,6 +838,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) testViewChannelPermissions(v s.Require().NoError(err) s.Require().Len(response.Communities(), 1) s.Require().True(s.owner.communitiesManager.IsChannelEncrypted(community.IDString(), chat.ID)) + s.Require().True(response.Communities()[0].ChannelHasPermissions(chat.CommunityChatID())) err = <-waitOnBobToBeKickedFromChannel s.Require().NoError(err) @@ -956,6 +859,9 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) testViewChannelPermissions(v if c == nil { return false } + if !c.ChannelEncrypted(chat.CommunityChatID()) { + return false + } channel := c.Chats()[chat.CommunityChatID()] return channel != nil && len(channel.Members) == 0 }, @@ -965,6 +871,8 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) testViewChannelPermissions(v // bob should not be in the bloom filter list community, err = s.bob.communitiesManager.GetByID(community.ID()) + s.Require().True(s.bob.communitiesManager.IsChannelEncrypted(community.IDString(), chat.ID)) + s.Require().True(community.ChannelHasPermissions(chat.CommunityChatID())) s.Require().NoError(err) s.Require().False(community.IsMemberLikelyInChat(chat.CommunityChatID())) @@ -1078,6 +986,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestAnnouncementsChannelPerm response, err := s.owner.CreateCommunityTokenPermission(&channelPermissionRequest) s.Require().NoError(err) s.Require().Len(response.Communities(), 1) + s.Require().True(response.Communities()[0].ChannelHasPermissions(chat.CommunityChatID())) s.Require().False(s.owner.communitiesManager.IsChannelEncrypted(community.IDString(), chat.ID)) // bob should be in the bloom filter list since everyone has access to readonly channels @@ -1094,32 +1003,124 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestAnnouncementsChannelPerm _, err = WaitOnMessengerResponse( s.bob, func(r *MessengerResponse) bool { - c, err := s.bob.GetCommunityByID(community.ID()) - if err != nil { + if len(r.Communities()) == 0 || !r.Communities()[0].ChannelHasPermissions(chat.CommunityChatID()) { return false } + c := r.Communities()[0] if c == nil { return false } channel := c.Chats()[chat.CommunityChatID()] + // The channel is not encrypted. Should have one member + // and bob should be a viewer + if channel == nil || len(channel.Members) != 1 { + return false + } + return c.IsMemberInChat(s.bob.IdentityPublicKey(), chat.CommunityChatID()) && + c.IsMemberLikelyInChat(chat.CommunityChatID()) + }, + "no community that satisfies criteria", + ) + s.Require().NoError(err) + + // bob can't post + msg := &common.Message{ + ChatMessage: &protobuf.ChatMessage{ + ChatId: chat.ID, + ContentType: protobuf.ChatMessage_TEXT_PLAIN, + Text: "I can't post on read-only channel", + }, + } + + _, err = s.bob.SendChatMessage(context.Background(), msg) + s.Require().Error(err) + s.Require().Contains(err.Error(), "can't post") + + // owner can post + msg = s.sendChatMessage(s.owner, chat.ID, "hello on announcements channel") + // bob can read the message + response, err = WaitOnMessengerResponse( + s.bob, + func(r *MessengerResponse) bool { + _, ok := r.messages[msg.ID] + return ok + }, + "no messages", + ) + s.Require().NoError(err) + s.Require().Len(response.Messages(), 1) + s.Require().Equal(msg.Text, response.Messages()[0].Text) + + // alice joins the community + s.advertiseCommunityTo(community, s.alice) + s.joinCommunity(community, s.alice) + + // setup view and post channel permission + channelPermissionRequest = requests.CreateCommunityTokenPermission{ + CommunityID: community.ID(), + Type: protobuf.CommunityTokenPermission_CAN_VIEW_AND_POST_CHANNEL, + ChatIds: []string{chat.ID}, + TokenCriteria: []*protobuf.TokenCriteria{ + { + Type: protobuf.CommunityTokenType_ERC20, + ContractAddresses: map[uint64]string{testChainID1: "0x124"}, + Symbol: "TEST2", + AmountInWei: "200000000000000000000", + Decimals: uint64(18), + }, + }, + } + + response, err = s.owner.CreateCommunityTokenPermission(&channelPermissionRequest) + s.Require().NoError(err) + s.Require().Len(response.Communities(), 1) + s.Require().True(response.Communities()[0].ChannelHasPermissions(chat.CommunityChatID())) + s.Require().False(s.owner.communitiesManager.IsChannelEncrypted(community.IDString(), chat.ID)) + + // make alice satisfy channel criteria + s.makeAddressSatisfyTheCriteria(testChainID1, aliceAddress1, channelPermissionRequest.TokenCriteria[0]) + // force owner to reevaluate channel members + // in production it will happen automatically, by periodic check + err = s.owner.communitiesManager.ForceMembersReevaluation(community.ID()) + + // alice receives community changes + _, err = WaitOnMessengerResponse( + s.alice, + func(r *MessengerResponse) bool { + if len(r.Communities()) == 0 || !r.Communities()[0].ChannelHasPermissions(chat.CommunityChatID()) { + return false + } + c := r.Communities()[0] + if c == nil { + return false + } + channel := c.Chats()[chat.CommunityChatID()] + // The channel is not encrypted, but has permissions. Should have 2 members + // Alice - poster, bob - viewer if channel == nil || len(channel.Members) != 2 { return false } - member := channel.Members[s.bob.IdentityPublicKeyString()] - return member != nil && member.ChannelRole == protobuf.CommunityMember_CHANNEL_ROLE_VIEWER + + if channel.Members[s.alice.IdentityPublicKeyString()].ChannelRole != protobuf.CommunityMember_CHANNEL_ROLE_POSTER { + return false + } + + return c.IsMemberLikelyInChat(chat.CommunityChatID()) }, "no community that satisfies criteria", ) - s.Require().NoError(err) - // bob should be in the bloom filter list - community, err = s.bob.communitiesManager.GetByID(community.ID()) + community, err = s.alice.communitiesManager.GetByID(community.ID()) s.Require().NoError(err) - s.Require().True(community.IsMemberLikelyInChat(chat.CommunityChatID())) + chatID := strings.TrimPrefix(chat.ID, community.IDString()) + members := community.Chats()[chatID].Members + s.Require().Len(members, 2) + // confirm that member is a viewer and not a poster + s.Require().Equal(protobuf.CommunityMember_CHANNEL_ROLE_POSTER, members[s.alice.IdentityPublicKeyString()].ChannelRole) // bob can't post - msg := &common.Message{ + msg = &common.Message{ ChatMessage: &protobuf.ChatMessage{ ChatId: chat.ID, ContentType: protobuf.ChatMessage_TEXT_PLAIN, @@ -1130,6 +1131,44 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestAnnouncementsChannelPerm _, err = s.bob.SendChatMessage(context.Background(), msg) s.Require().Error(err) s.Require().Contains(err.Error(), "can't post") + + // alice can post + msg = &common.Message{ + ChatMessage: &protobuf.ChatMessage{ + ChatId: chat.ID, + ContentType: protobuf.ChatMessage_TEXT_PLAIN, + Text: "I can post on read-only channel", + }, + } + + _, err = s.alice.SendChatMessage(context.Background(), msg) + s.Require().NoError(err) + + // owner can read the message + response, err = WaitOnMessengerResponse( + s.owner, + func(r *MessengerResponse) bool { + _, ok := r.messages[msg.ID] + return ok + }, + "no messages", + ) + s.Require().NoError(err) + s.Require().Len(response.Messages(), 1) + s.Require().Equal(msg.Text, response.Messages()[0].Text) + + // bob can read the message + response, err = WaitOnMessengerResponse( + s.bob, + func(r *MessengerResponse) bool { + _, ok := r.messages[msg.ID] + return ok + }, + "no messages", + ) + s.Require().NoError(err) + s.Require().Len(response.Messages(), 1) + s.Require().Equal(msg.Text, response.Messages()[0].Text) } func (s *MessengerCommunitiesTokenPermissionsSuite) TestSearchMessageinPermissionedChannel() { @@ -1196,7 +1235,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestSearchMessageinPermissio waitOnBobToBeKickedFromChannel := waitOnCommunitiesEvent(s.owner, func(sub *communities.Subscription) bool { channel, ok := sub.Community.Chats()[chat.CommunityChatID()] - return ok && len(channel.Members) == 1 + return ok && len(channel.Members) == 0 }) waitOnChannelToBeRekeyedOnceBobIsKicked := s.waitOnKeyDistribution(func(sub *CommunityAndKeyActions) bool { action, ok := sub.keyActions.ChannelKeysActions[chat.CommunityChatID()] @@ -1243,6 +1282,9 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestSearchMessageinPermissio if c == nil { return false } + if !c.ChannelEncrypted(chat.CommunityChatID()) { + return false + } channel := c.Chats()[chat.CommunityChatID()] return channel != nil && len(channel.Members) == 0 }, @@ -1285,9 +1327,15 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestMemberRoleGetUpdatedWhen s.Require().Len(response.Messages(), 1) s.Require().Equal(msg.Text, response.Messages()[0].Text) + community, err = s.bob.communitiesManager.GetByID(community.ID()) + s.Require().NoError(err) + s.Require().Len(community.Chats(), 1) + s.Require().Len(community.Chats()[chat.CommunityChatID()].Members, 0) + s.Require().True(community.IsMemberInChat(community.MemberIdentity(), chat.CommunityChatID())) + waitOnBobToBeKickedFromChannel := waitOnCommunitiesEvent(s.owner, func(sub *communities.Subscription) bool { channel, ok := sub.Community.Chats()[chat.CommunityChatID()] - return ok && len(channel.Members) == 1 + return ok && len(channel.Members) == 0 && !sub.Community.IsMemberInChat(community.MemberIdentity(), chat.CommunityChatID()) }) waitOnChannelToBeRekeyedOnceBobIsKicked := s.waitOnKeyDistribution(func(sub *CommunityAndKeyActions) bool { action, ok := sub.keyActions.ChannelKeysActions[chat.CommunityChatID()] @@ -1315,6 +1363,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestMemberRoleGetUpdatedWhen s.Require().Len(response.Communities(), 1) s.Require().Len(response.CommunityChanges[0].TokenPermissionsAdded, 1) s.Require().True(s.owner.communitiesManager.IsChannelEncrypted(community.IDString(), chat.ID)) + s.Require().True(response.Communities()[0].ChannelHasPermissions(chat.CommunityChatID())) err = <-waitOnBobToBeKickedFromChannel s.Require().NoError(err) @@ -1336,7 +1385,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestMemberRoleGetUpdatedWhen return false } channel := c.Chats()[chat.CommunityChatID()] - return channel != nil && len(channel.Members) == 0 + return channel != nil && len(channel.Members) == 0 && !c.IsMemberInChat(c.MemberIdentity(), chat.CommunityChatID()) }, "no community that satisfies criteria", ) @@ -1367,7 +1416,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestMemberRoleGetUpdatedWhen s.Require().NoError(err) chatID := strings.TrimPrefix(chat.ID, community.IDString()) members := community.Chats()[chatID].Members - s.Require().Len(members, 2) + s.Require().Len(members, 1) // confirm that member is a viewer and not a poster s.Require().Equal(protobuf.CommunityMember_CHANNEL_ROLE_VIEWER, members[s.bob.IdentityPublicKeyString()].ChannelRole) @@ -1430,7 +1479,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestMemberRoleGetUpdatedWhen s.Require().NoError(err) members = community.Chats()[chatID].Members - s.Require().Len(members, 2) + s.Require().Len(members, 1) // confirm that member is now a poster s.Require().Equal(protobuf.CommunityMember_CHANNEL_ROLE_POSTER, members[s.bob.IdentityPublicKeyString()].ChannelRole) @@ -1451,7 +1500,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestMemberRoleGetUpdatedWhen return false } members = chats[chat.ID].Members - return len(members) == 2 && members[s.bob.myHexIdentity()] != nil && members[s.bob.myHexIdentity()].ChannelRole == protobuf.CommunityMember_CHANNEL_ROLE_POSTER + return len(members) == 1 && members[s.bob.myHexIdentity()] != nil && members[s.bob.myHexIdentity()].ChannelRole == protobuf.CommunityMember_CHANNEL_ROLE_POSTER }, "bob never got post permissions", ) @@ -1845,7 +1894,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestResendEncryptionKeyOnBac s.Require().NoError(err) // bob receives community changes - // channel members should not be empty, + // channel members should be empty, // this info is available only to channel members with encryption key _, err = WaitOnMessengerResponse( s.bob, @@ -1857,8 +1906,13 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestResendEncryptionKeyOnBac if c == nil { return false } + + if !c.ChannelEncrypted(chat.CommunityChatID()) { + return false + } + channel := c.Chats()[chat.CommunityChatID()] - if channel != nil && len(channel.Members) < 2 { + if channel != nil && len(channel.Members) == 0 { return false } @@ -1894,12 +1948,12 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestResendEncryptionKeyOnBac backupMessage, err := s.bob.backupCommunity(community, clock) s.Require().NoError(err) - err = s.bob.HandleBackup(s.bob.buildMessageState(), backupMessage, nil) - s.Require().NoError(err) + errs := s.bob.handleLocalBackupCommunities(s.bob.buildMessageState(), backupMessage.Communities) + s.Require().Len(errs, 0, fmt.Sprintf("expected no errors while handling backup communities. Errors: %v", errs)) // regenerate key for the channel in order to check that owner will send keys // on bob request from `HandleBackup` - _, err = s.owner.encryptor.GenerateHashRatchetKey([]byte(community.IDString() + chat.CommunityChatID())) + err = s.owner.messaging.GenerateHashRatchetKey([]byte(community.IDString() + chat.CommunityChatID())) s.Require().NoError(err) testCommunitiesKeyDistributor, ok := s.owner.communitiesKeyDistributor.(*TestCommunitiesKeyDistributor) diff --git a/protocol/messenger.go b/protocol/messenger.go index 8cf4c83e43d..9fa4848e7cc 100644 --- a/protocol/messenger.go +++ b/protocol/messenger.go @@ -20,45 +20,33 @@ import ( "go.uber.org/zap" "golang.org/x/time/rate" - datasyncnode "github.com/status-im/mvds/node" - gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/status-im/status-go/account" - "github.com/status-im/status-go/appmetrics" gocommon "github.com/status-im/status-go/common" utils "github.com/status-im/status-go/common" "github.com/status-im/status-go/connection" "github.com/status-im/status-go/contracts" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/images" "github.com/status-im/status-go/internal/newsfeed" "github.com/status-im/status-go/messaging" messagingtypes "github.com/status-im/status-go/messaging/types" - "github.com/status-im/status-go/metrics/wakumetrics" multiaccountscommon "github.com/status-im/status-go/multiaccounts/common" "github.com/status-im/status-go/multiaccounts" "github.com/status-im/status-go/multiaccounts/accounts" "github.com/status-im/status-go/multiaccounts/settings" - "github.com/status-im/status-go/protocol/anonmetrics" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" - "github.com/status-im/status-go/protocol/encryption" - "github.com/status-im/status-go/protocol/encryption/multidevice" - "github.com/status-im/status-go/protocol/encryption/sharedsecret" "github.com/status-im/status-go/protocol/ens" "github.com/status-im/status-go/protocol/identity/alias" "github.com/status-im/status-go/protocol/identity/identicon" "github.com/status-im/status-go/protocol/peersyncing" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/pushnotificationclient" - "github.com/status-im/status-go/protocol/pushnotificationserver" "github.com/status-im/status-go/protocol/requests" - "github.com/status-im/status-go/protocol/sqlite" - "github.com/status-im/status-go/protocol/storenodes" v1protocol "github.com/status-im/status-go/protocol/v1" "github.com/status-im/status-go/protocol/verification" "github.com/status-im/status-go/server" @@ -71,11 +59,15 @@ import ( "github.com/status-im/status-go/services/wallet/token" "github.com/status-im/status-go/signal" - wakutypes "github.com/status-im/status-go/waku/types" - _ "github.com/mmcdole/gofeed" ) +type PushNotificationServer interface { + HandlePushNotificationQuery(publicKey *ecdsa.PublicKey, messageID []byte, query *protobuf.PushNotificationQuery) error + HandlePushNotificationRequest(publicKey *ecdsa.PublicKey, messageID []byte, request *protobuf.PushNotificationRequest) error + HandlePushNotificationRegistration(publicKey *ecdsa.PublicKey, payload []byte) error +} + const ( PubKeyStringLength = 132 @@ -95,7 +87,7 @@ const communityAdvertiseIntervalSecond int64 = 24 * 60 * 60 // messageCacheIntervalMs is how long we should keep processed messages in the cache, in ms var messageCacheIntervalMs uint64 = 1000 * 60 * 60 * 48 -// Messenger is a entity managing chats and messages. +// Messenger is an entity managing chats and messages. // It acts as a bridge between the application and encryption // layers. // It needs to expose an interface to manage installations @@ -105,19 +97,17 @@ var messageCacheIntervalMs uint64 = 1000 * 60 * 60 * 48 type Messenger struct { config *config identity *ecdsa.PrivateKey + signer communities.MessageSigner messaging *messaging.API + messagingPersistence *messagingPersistence persistence *sqlitePersistence - encryptor *encryption.Protocol - sender *common.MessageSender ensVerifier *ens.Verifier - anonMetricsClient *anonmetrics.Client - anonMetricsServer *anonmetrics.Server pushNotificationClient *pushnotificationclient.Client - pushNotificationServer *pushnotificationserver.Server + pushNotificationServer PushNotificationServer communitiesManager *communities.Manager archiveManager communities.ArchiveService communitiesKeyDistributor communities.KeyDistributor - accountsManager account.Manager + accountsManager AccountsManager mentionsManager *MentionManager storeNodeRequestsManager *StoreNodeRequestManager logger *zap.Logger @@ -137,7 +127,6 @@ type Messenger struct { allInstallations *installationMap modifiedInstallations *stringBoolMap installationID string - communityStorenodes *storenodes.CommunityStorenodes database *sql.DB multiAccounts *multiaccounts.Database settings *accounts.Database @@ -161,7 +150,6 @@ type Messenger struct { } connectionState connection.State - wakuMetricsHandler *wakumetrics.Client contractMaker *contracts.ContractMaker verificationDatabase *verification.Persistence savedAddressesManager *wallet.SavedAddressesManager @@ -174,16 +162,14 @@ type Messenger struct { // flag to disable checking #hasPairedDevices localPairing bool - // flag to enable backedup messages processing, false by default - processBackedupMessages bool communityTokensService communities.CommunityTokensServiceInterface // used to track dispatched messages - dispatchMessageTestCallback func(common.RawMessage) + dispatchMessageTestCallback func(messagingtypes.RawMessage) // used to track unhandled messages - unhandledMessagesTracker func(*v1protocol.StatusMessage, error) + unhandledMessagesTracker func(*messagingtypes.Message, error) // enables control over chat messages iteration retrievedMessagesIteratorFactory func(map[messagingtypes.ChatFilter][]*messagingtypes.ReceivedMessage) MessagesIterator @@ -192,10 +178,6 @@ type Messenger struct { peersyncingOffers map[string]uint64 peersyncingRequests map[string]uint64 - mvdsStatusChangeEvent chan datasyncnode.PeerStatusChangeEvent - - backedUpFetchingStatus *BackupFetchingStatus - newsFeedManager *newsfeed.NewsFeedManager } @@ -284,11 +266,9 @@ func (interceptor EnvelopeEventsInterceptor) MailServerRequestExpired(hash types } func NewMessenger( - nodeName string, identity *ecdsa.PrivateKey, - waku wakutypes.Waku, + messaging *messaging.API, installationID string, - version string, opts ...Option, ) (*Messenger, error) { var messenger *Messenger @@ -325,80 +305,6 @@ func NewMessenger( return nil, err } } - - // Apply migrations for all components. - err := sqlite.Migrate(database) - if err != nil { - return nil, errors.Wrap(err, "failed to apply migrations") - } - - messaging, err := messaging.NewCore( - waku, - identity, - common.NewMessagingPersistence(database), - messaging.WithLogger(logger), - messaging.WithEnvelopeEventsConfig(c.envelopeEventsConfig), - ) - if err != nil { - return nil, errors.Wrap(err, "failed to create messaging core") - } - - // Initialize encryption layer. - encryptionProtocol := encryption.New( - database, - installationID, - logger, - ) - - sender, err := common.NewMessageSender( - identity, - database, - messaging.API(), - encryptionProtocol, - logger, - c.featureFlags, - ) - if err != nil { - return nil, errors.Wrap(err, "failed to create messageSender") - } - - // Initialise anon metrics client - var anonMetricsClient *anonmetrics.Client - if c.anonMetricsClientConfig != nil && - c.anonMetricsClientConfig.ShouldSend && - c.anonMetricsClientConfig.Active == anonmetrics.ActiveClientPhrase { - - anonMetricsClient = anonmetrics.NewClient(sender) - anonMetricsClient.Config = c.anonMetricsClientConfig - anonMetricsClient.Identity = identity - anonMetricsClient.DB = appmetrics.NewDB(database) - anonMetricsClient.Logger = logger - } - - // Initialise anon metrics server - var anonMetricsServer *anonmetrics.Server - if c.anonMetricsServerConfig != nil && - c.anonMetricsServerConfig.Enabled && - c.anonMetricsServerConfig.Active == anonmetrics.ActiveServerPhrase { - - server, err := anonmetrics.NewServer(c.anonMetricsServerConfig.PostgresURI) - if err != nil { - return nil, errors.Wrap(err, "failed to create anonmetrics.Server") - } - - anonMetricsServer = server - anonMetricsServer.Config = c.anonMetricsServerConfig - anonMetricsServer.Logger = logger - } - - // Initialize push notification server - var pushNotificationServer *pushnotificationserver.Server - if c.pushNotificationServerConfig != nil && c.pushNotificationServerConfig.Enabled { - c.pushNotificationServerConfig.Identity = identity - pushNotificationServerPersistence := pushnotificationserver.NewSQLitePersistence(database) - pushNotificationServer = pushnotificationserver.New(c.pushNotificationServerConfig, pushNotificationServerPersistence, sender) - } - // Initialize push notification client pushNotificationClientPersistence := pushnotificationclient.NewPersistence(database) pushNotificationClientConfig := c.pushNotificationClientConfig @@ -412,12 +318,10 @@ func NewMessenger( pushNotificationClientConfig.Logger = logger pushNotificationClientConfig.InstallationID = installationID - pushNotificationClient := pushnotificationclient.New(pushNotificationClientPersistence, pushNotificationClientConfig, sender, sqlitePersistence) - - ensVerifier := ens.New(logger, messaging.API(), database, c.verifyENSURL, c.verifyENSContractAddress) + pushNotificationClient := pushnotificationclient.New(pushNotificationClientPersistence, pushNotificationClientConfig, messaging, sqlitePersistence) managerOptions := []communities.ManagerOption{ - communities.WithAccountManager(c.accountsManager), + communities.WithMessageSigner(c.signer), } var walletAPI *wallet.API @@ -431,12 +335,8 @@ func NewMessenger( if c.tokenManager != nil { managerOptions = append(managerOptions, communities.WithTokenManager(c.tokenManager)) } else if c.rpcClient != nil { - tokenManager := token.NewTokenManager(c.walletDb, c.rpcClient, community.NewManager(database, c.httpServer, nil), c.rpcClient.NetworkManager, database, c.httpServer, nil, nil, nil, token.NewPersistence(c.walletDb)) - managerOptions = append(managerOptions, communities.WithTokenManager(communities.NewDefaultTokenManager(tokenManager, c.rpcClient.NetworkManager))) - } - - if c.walletConfig != nil { - managerOptions = append(managerOptions, communities.WithWalletConfig(c.walletConfig)) + tokenManager := token.NewTokenManager(c.walletDb, c.rpcClient, community.NewManager(database, c.httpServer, nil), c.rpcClient.GetNetworkManager(), database, c.httpServer, nil, nil, nil, token.NewPersistence(c.walletDb)) + managerOptions = append(managerOptions, communities.WithTokenManager(communities.NewDefaultTokenManager(tokenManager, c.rpcClient.GetNetworkManager()))) } if c.communityTokensService != nil { @@ -446,20 +346,18 @@ func NewMessenger( managerOptions = append(managerOptions, c.communityManagerOptions...) communitiesKeyDistributor := &CommunitiesKeyDistributorImpl{ - sender: sender, - encryptor: encryptionProtocol, + messaging: messaging, } communitiesManager, err := communities.NewManager( identity, installationID, database, - encryptionProtocol, logger, - ensVerifier, + c.ensVerifier, c.communityTokensService, - messaging.API(), - messaging.API(), + messaging, + messaging, communitiesKeyDistributor, c.httpServer, managerOptions..., @@ -472,9 +370,8 @@ func NewMessenger( TorrentConfig: c.torrentConfig, Logger: logger, Persistence: communitiesManager.GetPersistence(), - Messaging: messaging.API(), + Messaging: messaging, Identity: identity, - Encryptor: encryptionProtocol, Publisher: communitiesManager, } @@ -482,9 +379,6 @@ func NewMessenger( // "communities/manager_archive.go" version of this function based on the build instructions for those files. // See those file for more details. archiveManager := communities.NewArchiveManager(amc) - if err != nil { - return nil, err - } settings, err := accounts.NewDB(database) if err != nil { @@ -498,44 +392,22 @@ func NewMessenger( return nil, fmt.Errorf("failed to build contact of ourself: %w", err) } - var wakuMetricsHandler *wakumetrics.Client - if c.telemetryServerURL != "" { - options := []wakumetrics.TelemetryClientOption{ - wakumetrics.WithPeerID(waku.PeerID().String()), - } - wakuMetricsHandler, err = wakumetrics.NewClient(options...) - if err != nil { - return nil, err - } - if c.wakuService != nil { - c.wakuService.SetMetricsHandler(wakuMetricsHandler) - } - sender.SetMetricsHandler(wakuMetricsHandler) - err = wakuMetricsHandler.RegisterWithRegistry() - if err != nil { - return nil, err - } - } - ctx, cancel := context.WithCancel(context.Background()) messenger = &Messenger{ config: &c, identity: identity, - messaging: messaging.API(), + messaging: messaging, + messagingPersistence: NewMessagingPersistence(database), persistence: sqlitePersistence, - encryptor: encryptionProtocol, - sender: sender, - anonMetricsClient: anonMetricsClient, - anonMetricsServer: anonMetricsServer, - wakuMetricsHandler: wakuMetricsHandler, communityTokensService: c.communityTokensService, pushNotificationClient: pushNotificationClient, - pushNotificationServer: pushNotificationServer, + pushNotificationServer: c.pushNotificationServer, communitiesManager: communitiesManager, communitiesKeyDistributor: communitiesKeyDistributor, archiveManager: archiveManager, accountsManager: c.accountsManager, - ensVerifier: ensVerifier, + ensVerifier: c.ensVerifier, + signer: c.signer, featureFlags: c.featureFlags, systemMessagesTranslations: c.systemMessagesTranslations, allChats: new(chatMap), @@ -551,13 +423,11 @@ func NewMessenger( database: database, multiAccounts: c.multiAccount, settings: settings, - peersyncing: peersyncing.New(peersyncing.Config{Database: database, Timesource: messaging.API()}), + peersyncing: peersyncing.New(peersyncing.Config{Database: database, Timesource: messaging}), peersyncingOffers: make(map[string]uint64), peersyncingRequests: make(map[string]uint64), - mvdsStatusChangeEvent: make(chan datasyncnode.PeerStatusChangeEvent, 5), verificationDatabase: verification.NewPersistence(database), mailserversDatabase: c.mailserversDatabase, - communityStorenodes: storenodes.NewCommunityStorenodes(storenodes.NewDB(database), logger), account: c.account, quit: make(chan struct{}), ctx: ctx, @@ -572,28 +442,9 @@ func NewMessenger( browserDatabase: c.browserDatabase, httpServer: c.httpServer, shutdownTasks: []func() error{ - ensVerifier.Stop, pushNotificationClient.Stop, communitiesManager.Stop, archiveManager.Stop, - encryptionProtocol.Stop, - wakumetrics.UnregisterMetrics, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - err := messaging.API().ResetChatFilters(ctx) - if err != nil { - logger.Warn("could not reset filters", zap.Error(err)) - } - // We don't want to thrown an error in this case, this is a soft - // fail - return nil - }, - messaging.API().Stop, - func() error { sender.Stop(); return nil }, - // Currently this often fails, seems like it's safe to ignore them - // https://github.com/uber-go/zap/issues/328 - func() error { _ = logger.Sync; return nil }, database.Close, }, logger: logger, @@ -632,11 +483,8 @@ func NewMessenger( messenger.shutdownTasks = append(messenger.shutdownTasks, csvFile.Close) } - if anonMetricsClient != nil { - messenger.shutdownTasks = append(messenger.shutdownTasks, anonMetricsClient.Stop) - } - if anonMetricsServer != nil { - messenger.shutdownTasks = append(messenger.shutdownTasks, anonMetricsServer.Stop) + if c.ensVerifier != nil { + messenger.shutdownTasks = append(messenger.shutdownTasks, c.ensVerifier.Stop) } if c.envelopeEventsConfig != nil { @@ -650,10 +498,6 @@ func NewMessenger( return messenger, nil } -func (m *Messenger) EnableBackedupMessagesProcessing() { - m.processBackedupMessages = true -} - func (m *Messenger) processSentMessage(id string) error { if m.connectionState.Offline { return errors.New("Can't mark message as sent while offline") @@ -663,7 +507,7 @@ func (m *Messenger) processSentMessage(id string) error { // If we have no raw message, we create a temporary one, so that // the sent status is preserved if err == sql.ErrNoRows || rawMessage == nil { - rawMessage = &common.RawMessage{ + rawMessage = &messagingtypes.RawMessage{ ID: id, MessageType: protobuf.ApplicationMetadataMessage_CHAT_MESSAGE, } @@ -717,12 +561,6 @@ func (m *Messenger) Start() (*MessengerResponse, error) { } m.logger.Info("starting messenger", zap.String("identity", types.EncodeHex(crypto.FromECDSAPub(&m.identity.PublicKey)))) - // Start push notification server - if m.pushNotificationServer != nil { - if err := m.pushNotificationServer.Start(); err != nil { - return nil, err - } - } // Start push notification client if m.pushNotificationClient != nil { @@ -733,42 +571,21 @@ func (m *Messenger) Start() (*MessengerResponse, error) { } } - // Start anonymous metrics client - if m.anonMetricsClient != nil { - if err := m.anonMetricsClient.Start(); err != nil { + var ensSubscription chan []*ens.VerificationRecord + if m.ensVerifier != nil { + ensSubscription = m.ensVerifier.Subscribe() + + // Subscribe + if err := m.ensVerifier.Start(); err != nil { return nil, err } } - ensSubscription := m.ensVerifier.Subscribe() - - // Subscrbe - if err := m.ensVerifier.Start(); err != nil { - return nil, err - } - if err := m.communitiesManager.Start(); err != nil { return nil, err } - // set shared secret handles - m.sender.SetHandleSharedSecrets(m.handleSharedSecrets) - if err := m.sender.StartDatasync(m.mvdsStatusChangeEvent, m.sendDataSync); err != nil { - return nil, err - } - - subscriptions, err := m.encryptor.Start(m.identity) - if err != nil { - return nil, err - } - - // handle stored shared secrets - err = m.handleSharedSecrets(subscriptions.SharedSecrets) - if err != nil { - return nil, err - } - - m.handleEncryptionLayerSubscriptions(subscriptions) + m.handleEncryptionLayerSubscriptions(m.messaging.EncryptionSubscriptions()) m.handleCommunitiesSubscription(m.communitiesManager.Subscribe()) m.handleCommunitiesHistoryArchivesSubscription(m.communitiesManager.Subscribe()) m.updateCommunitiesActiveMembersPeriodically() @@ -783,9 +600,6 @@ func (m *Messenger) Start() (*MessengerResponse, error) { m.watchPendingCommunityRequestToJoin() m.broadcastLatestUserStatus() m.timeoutAutomaticStatusUpdates() - if !m.config.featureFlags.DisableCheckingForBackup { - m.startBackupLoop() - } if !m.config.featureFlags.DisableAutoMessageLoop { err = m.startAutoMessageLoop() if err != nil { @@ -799,8 +613,6 @@ func (m *Messenger) Start() (*MessengerResponse, error) { if m.config.codeControlFlags.CuratedCommunitiesUpdateLoopEnabled { m.startCuratedCommunitiesUpdateLoop() } - m.startMessageSegmentsCleanupLoop() - m.startHashRatchetEncryptedMessagesCleanupLoop() m.startRequestMissingCommunityChannelsHRKeysLoop() if err := m.cleanTopics(); err != nil { @@ -808,17 +620,13 @@ func (m *Messenger) Start() (*MessengerResponse, error) { } response := &MessengerResponse{} - response.Mailservers, err = m.AllMailservers() + response.StoreNodes, err = m.AllMailservers() if err != nil { return nil, err } m.messaging.SetStorenodeConfigProvider(m) - if err := m.communityStorenodes.ReloadFromDB(); err != nil { - return nil, err - } - go m.checkForMissingMessagesLoop() go m.checkForStorenodeCycleSignals() @@ -885,18 +693,6 @@ func (m *Messenger) Start() (*MessengerResponse, error) { } } - if m.processBackedupMessages { - m.backedUpFetchingStatus = &BackupFetchingStatus{ - dataProgress: make(map[string]FetchingBackedUpDataTracking), - lastKnownMsgClock: 0, - fetchingCompleted: false, - } - err = m.startBackupFetchingTracking(response) - if err != nil { - return nil, err - } - } - if m.config.featureFlags.EnableNewsFeed { lastFetched, err := m.settings.NewsFeedLastFetchedTimestamp() if err != nil { @@ -1013,10 +809,16 @@ func (m *Messenger) handleConnectionChange(online bool) { } // Update ENS verifier - m.ensVerifier.SetOnline(online) + if m.ensVerifier != nil { + m.ensVerifier.SetOnline(online) + } } func (m *Messenger) Online() bool { + if !m.started { + return false + } + if m.config.onlineChecker != nil { return m.config.onlineChecker() } @@ -1072,16 +874,16 @@ func (m *Messenger) publishContactCode() error { } contactCodeTopic := messaging.ContactCodeTopic(&m.identity.PublicKey) - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: contactCodeTopic, MessageType: protobuf.ApplicationMetadataMessage_CONTACT_CODE_ADVERTISEMENT, Payload: payload, - Priority: &common.LowPriority, + Priority: &messagingtypes.LowPriority, } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, err = m.sender.SendPublic(ctx, contactCodeTopic, rawMessage) + _, err = m.messaging.SendPublic(ctx, contactCodeTopic, rawMessage) if err != nil { m.logger.Warn("failed to send a contact code", zap.Error(err)) } @@ -1093,7 +895,7 @@ func (m *Messenger) publishContactCode() error { for _, community := range joinedCommunities { rawMessage.LocalChatID = community.MemberUpdateChannelID() rawMessage.PubsubTopic = community.PubsubTopic() - _, err = m.sender.SendPublic(ctx, rawMessage.LocalChatID, rawMessage) + _, err = m.messaging.SendPublic(ctx, rawMessage.LocalChatID, rawMessage) if err != nil { return err } @@ -1179,16 +981,16 @@ func (m *Messenger) handleStandaloneChatIdentity(chat *Chat) error { return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, MessageType: protobuf.ApplicationMetadataMessage_CHAT_IDENTITY, Payload: payload, - Priority: &common.LowPriority, + Priority: &messagingtypes.LowPriority, } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if chat.ChatType == ChatTypePublic { - _, err = m.sender.SendPublic(ctx, chat.ID, rawMessage) + _, err = m.messaging.SendPublic(ctx, chat.ID, rawMessage) if err != nil { return err } @@ -1197,7 +999,7 @@ func (m *Messenger) handleStandaloneChatIdentity(chat *Chat) error { if err != nil { return err } - _, err = m.sender.SendPrivate(ctx, pk, &rawMessage) + _, err = m.messaging.SendPrivate(ctx, pk, &rawMessage) if err != nil { return err } @@ -1405,23 +1207,8 @@ func (m *Messenger) attachIdentityImagesToChatIdentity(context ChatContext, ci * return nil } -// handleSharedSecrets process the negotiated secrets received from the encryption layer -func (m *Messenger) handleSharedSecrets(secrets []*sharedsecret.Secret) error { - for _, secret := range secrets { - fSecret := types.NegotiatedSecret{ - PublicKey: secret.Identity, - Key: secret.Key, - } - _, err := m.messaging.ProcessNegotiatedSecret(fSecret) - if err != nil { - return err - } - } - return nil -} - // handleInstallations adds the installations in the installations map -func (m *Messenger) handleInstallations(installations []*multidevice.Installation) { +func (m *Messenger) handleInstallations(installations []*messagingtypes.Installation) { for _, installation := range installations { if installation.Identity == contactIDFromPublicKey(&m.identity.PublicKey) { if _, ok := m.allInstallations.Load(installation.ID); !ok { @@ -1433,7 +1220,7 @@ func (m *Messenger) handleInstallations(installations []*multidevice.Installatio } // handleEncryptionLayerSubscriptions handles events from the encryption layer -func (m *Messenger) handleEncryptionLayerSubscriptions(subscriptions *encryption.Subscriptions) { +func (m *Messenger) handleEncryptionLayerSubscriptions(subscriptions *messagingtypes.EncryptionSubscriptions) { go func() { defer gocommon.LogOnPanic() for { @@ -1732,7 +1519,7 @@ func (m *Messenger) handlePushNotificationClientRegistrations(c chan struct{}) { // Shutdown takes care of ensuring a clean shutdown of Messenger func (m *Messenger) Shutdown() (err error) { - if m == nil { + if m == nil || !m.started { return nil } @@ -1761,6 +1548,8 @@ func (m *Messenger) Shutdown() (err error) { } } } + m.started = false + _ = m.logger.Sync() return } @@ -1900,7 +1689,7 @@ func (m *Messenger) hasPairedDevices() bool { } var count int - m.allInstallations.Range(func(installationID string, installation *multidevice.Installation) (shouldContinue bool) { + m.allInstallations.Range(func(installationID string, installation *messagingtypes.Installation) (shouldContinue bool) { if installation.Enabled { count++ } @@ -1917,11 +1706,11 @@ func (m *Messenger) HasPairedDevices() bool { } // sendToPairedDevices will check if we have any paired devices and send to them if necessary -func (m *Messenger) sendToPairedDevices(ctx context.Context, spec common.RawMessage) error { +func (m *Messenger) sendToPairedDevices(ctx context.Context, spec messagingtypes.RawMessage) error { hasPairedDevices := m.hasPairedDevices() // We send a message to any paired device if hasPairedDevices { - _, err := m.sender.SendPrivate(ctx, &m.identity.PublicKey, &spec) + _, err := m.messaging.SendPrivate(ctx, &m.identity.PublicKey, &spec) if err != nil { return err } @@ -1929,11 +1718,11 @@ func (m *Messenger) sendToPairedDevices(ctx context.Context, spec common.RawMess return nil } -func (m *Messenger) dispatchPairInstallationMessage(ctx context.Context, spec common.RawMessage) (common.RawMessage, error) { +func (m *Messenger) dispatchPairInstallationMessage(ctx context.Context, spec messagingtypes.RawMessage) (messagingtypes.RawMessage, error) { var err error var id []byte - id, err = m.sender.SendPairInstallation(ctx, &m.identity.PublicKey, spec) + id, err = m.messaging.SendPairInstallation(ctx, &m.identity.PublicKey, spec) if err != nil { return spec, err @@ -1948,7 +1737,7 @@ func (m *Messenger) dispatchPairInstallationMessage(ctx context.Context, spec co return spec, nil } -func (m *Messenger) dispatchMessage(ctx context.Context, rawMessage common.RawMessage) (common.RawMessage, error) { +func (m *Messenger) dispatchMessage(ctx context.Context, rawMessage messagingtypes.RawMessage) (messagingtypes.RawMessage, error) { if rawMessage.ContentTopic == "" { rawMessage.ContentTopic = rawMessage.LocalChatID } @@ -1968,11 +1757,15 @@ func (m *Messenger) dispatchMessage(ctx context.Context, rawMessage common.RawMe return rawMessage, err } + if publicKey.Curve != m.identity.PublicKey.Curve { + return rawMessage, errors.New("public key curve does not match") + } + //SendPrivate will alter message identity and possibly datasyncid, so we save an unchanged //message for sending to paired devices later specCopyForPairedDevices := rawMessage if !common.IsPubKeyEqual(publicKey, &m.identity.PublicKey) || rawMessage.SkipEncryptionLayer { - id, err = m.sender.SendPrivate(ctx, publicKey, &rawMessage) + id, err = m.messaging.SendPrivate(ctx, publicKey, &rawMessage) if err != nil { return rawMessage, err @@ -1987,7 +1780,7 @@ func (m *Messenger) dispatchMessage(ctx context.Context, rawMessage common.RawMe case ChatTypePublic, ChatTypeProfile: logger.Debug("sending public message", zap.String("chatName", chat.Name)) - id, err = m.sender.SendPublic(ctx, rawMessage.ContentTopic, rawMessage) + id, err = m.messaging.SendPublic(ctx, rawMessage.ContentTopic, rawMessage) if err != nil { return rawMessage, err } @@ -2027,7 +1820,7 @@ func (m *Messenger) dispatchMessage(ctx context.Context, rawMessage common.RawMe } isEncrypted := isCommunityEncrypted || isChannelEncrypted if !isEncrypted { - id, err = m.sender.SendPublic(ctx, rawMessage.ContentTopic, rawMessage) + id, err = m.messaging.SendPublic(ctx, rawMessage.ContentTopic, rawMessage) if err != nil { return rawMessage, err } @@ -2043,7 +1836,7 @@ func (m *Messenger) dispatchMessage(ctx context.Context, rawMessage common.RawMe rawMessage.HashRatchetGroupID = rawMessage.CommunityID } - id, err = m.sender.SendCommunityMessage(ctx, &rawMessage) + id, err = m.messaging.SendCommunityMessage(ctx, &rawMessage) if err != nil { return rawMessage, err } @@ -2082,7 +1875,7 @@ func (m *Messenger) dispatchMessage(ctx context.Context, rawMessage common.RawMe rawMessage.MessageType = protobuf.ApplicationMetadataMessage_MEMBERSHIP_UPDATE_MESSAGE } - id, err = m.sender.SendGroup(ctx, rawMessage.Recipients, rawMessage) + id, err = m.messaging.SendGroup(ctx, rawMessage.Recipients, rawMessage) if err != nil { return rawMessage, err } @@ -2236,7 +2029,7 @@ func (m *Messenger) sendChatMessage(ctx context.Context, message *common.Message return nil, err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, SendPushNotification: m.featureFlags.PushNotifications, Payload: encodedMessage, @@ -2249,7 +2042,7 @@ func (m *Messenger) sendChatMessage(ctx context.Context, message *common.Message // This is not the best solution, probably it would be better to split // the sent status in a different table and join on query for messages, // but that's a much larger change and it would require an expensive migration of clients - rawMessage.BeforeDispatch = func(rawMessage *common.RawMessage) error { + rawMessage.BeforeDispatch = func(rawMessage *messagingtypes.RawMessage) error { if rawMessage.Sent { message.OutgoingStatus = common.OutgoingStatusSent @@ -2467,11 +2260,11 @@ func (m *Messenger) syncChat(ctx context.Context, chatToSync *Chat, rawMessageHa return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_CHAT, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2500,11 +2293,11 @@ func (m *Messenger) syncClearHistory(ctx context.Context, publicChat *Chat, rawM return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_CLEAR_HISTORY, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2532,11 +2325,11 @@ func (m *Messenger) syncChatRemoving(ctx context.Context, id string, rawMessageH return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_CHAT_REMOVED, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2566,11 +2359,11 @@ func (m *Messenger) syncContact(ctx context.Context, contact *Contact, rawMessag return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_INSTALLATION_CONTACT_V2, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2583,14 +2376,14 @@ func (m *Messenger) syncContact(ctx context.Context, contact *Contact, rawMessag } func (m *Messenger) propagateSyncInstallationCommunityWithHRKeys(msg *protobuf.SyncInstallationCommunity, c *communities.Community) error { - communityKeys, err := m.encryptor.GetAllHRKeysMarshaledV1(c.ID()) + communityKeys, err := m.messaging.GetAllHRKeysMarshaledV1(c.ID()) if err != nil { return err } msg.EncryptionKeysV1 = communityKeys communityAndChannelKeys := [][]byte{} - communityKeys, err = m.encryptor.GetAllHRKeysMarshaledV2(c.ID()) + communityKeys, err = m.messaging.GetAllHRKeysMarshaledV2(c.ID()) if err != nil { return err } @@ -2599,7 +2392,7 @@ func (m *Messenger) propagateSyncInstallationCommunityWithHRKeys(msg *protobuf.S } for channelID := range c.Chats() { - channelKeys, err := m.encryptor.GetAllHRKeysMarshaledV2([]byte(c.IDString() + channelID)) + channelKeys, err := m.messaging.GetAllHRKeysMarshaledV2([]byte(c.IDString() + channelID)) if err != nil { return err } @@ -2657,11 +2450,11 @@ func (m *Messenger) syncCommunity(ctx context.Context, community *communities.Co return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_INSTALLATION_COMMUNITY, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2694,11 +2487,11 @@ func (m *Messenger) SyncBookmark(ctx context.Context, bookmark *browsers.Bookmar return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_BOOKMARK, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) if err != nil { @@ -2745,7 +2538,7 @@ func (m *Messenger) saveEnsUsernameDetailProto(syncMessage *protobuf.SyncEnsUser return ud, nil } -func (m *Messenger) HandleSyncEnsUsernameDetail(state *ReceivedMessageState, syncMessage *protobuf.SyncEnsUsernameDetail, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncEnsUsernameDetail(state *ReceivedMessageState, syncMessage *protobuf.SyncEnsUsernameDetail, statusMessage *messagingtypes.Message) error { ud, err := m.saveEnsUsernameDetailProto(syncMessage) if err != nil { return err @@ -2767,11 +2560,11 @@ func (m *Messenger) syncEnsUsernameDetail(ctx context.Context, usernameDetail *e } _, chat := m.getLastClockWithRelatedChat() - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_ENS_USERNAME_DETAIL, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2796,11 +2589,11 @@ func (m *Messenger) syncAccountCustomizationColor(ctx context.Context, acc *mult return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_ACCOUNT_CUSTOMIZATION_COLOR, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = m.dispatchMessage(ctx, rawMessage) @@ -2824,11 +2617,11 @@ func (m *Messenger) SyncTrustedUser(ctx context.Context, publicKey string, ts ve return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_TRUSTED_USER, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2863,11 +2656,11 @@ func (m *Messenger) SyncVerificationRequest(ctx context.Context, vr *verificatio return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_VERIFICATION_REQUEST, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -2929,7 +2722,7 @@ func (m *Messenger) PublishMessengerResponse(response *MessengerResponse) { localnotifications.PushMessages(notifications) } -func (m *Messenger) GetStats() wakutypes.StatsSummary { +func (m *Messenger) GetStats() messagingtypes.TransportStats { return m.messaging.GetStats() } @@ -2945,7 +2738,7 @@ type CurrentMessageState struct { // PublicKey is the public key of the author of the message PublicKey *ecdsa.PublicKey - StatusMessage *v1protocol.StatusMessage + StatusMessage *messagingtypes.Message } type ReceivedMessageState struct { @@ -3187,7 +2980,7 @@ func (m *Messenger) handleImportedMessages(messagesToHandle map[messagingtypes.C for filter, messages := range messagesToHandle { for _, shhMessage := range messages { - handleMessageResponse, err := m.sender.HandleMessages(shhMessage) + handleMessageResponse, err := m.messaging.HandleReceivedMessages(shhMessage) if err != nil { logger.Info("failed to decode messages", zap.Error(err)) continue @@ -3361,7 +3154,7 @@ func (m *Messenger) handleRetrievedMessages(chatWithMessages map[messagingtypes. } } - handleMessagesResponse, err := m.sender.HandleMessages(shhMessage) + handleMessagesResponse, err := m.messaging.HandleReceivedMessages(shhMessage) if err != nil { logger.Info("failed to decode messages", zap.Error(err)) continue @@ -3373,13 +3166,11 @@ func (m *Messenger) handleRetrievedMessages(chatWithMessages map[messagingtypes. statusMessages := handleMessagesResponse.StatusMessages - if m.wakuMetricsHandler != nil { - m.wakuMetricsHandler.PushReceivedMessages(wakumetrics.ReceivedMessages{ - Filter: filter, - SHHMessage: shhMessage, - Messages: statusMessages, - }) - } + m.messaging.MetricsPushReceivedMessages(messagingtypes.ReceivedMessages{ + Filter: filter, + SHHMessage: shhMessage, + Messages: statusMessages, + }) err = m.handleDatasyncMetadata(handleMessagesResponse) if err != nil { @@ -3394,7 +3185,7 @@ func (m *Messenger) handleRetrievedMessages(chatWithMessages map[messagingtypes. publicKey := msg.SigPubKey() m.handleInstallations(msg.EncryptionLayer.Installations) - err := m.handleSharedSecrets(msg.EncryptionLayer.SharedSecrets) + err := m.messaging.HandleSharedSecrets(msg.EncryptionLayer.SharedSecrets) if err != nil { // log and continue, non-critical error logger.Warn("failed to handle shared secrets") @@ -4160,11 +3951,11 @@ func (m *Messenger) syncChatMessagesRead(ctx context.Context, chatID string, clo return err } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_CHAT_MESSAGES_READ, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err = rawMessageHandler(ctx, rawMessage) @@ -4455,11 +4246,11 @@ func (m *Messenger) RequestTransaction(ctx context.Context, chatID, value, contr if err != nil { return nil, err } - resendType := common.ResendTypeRawMessage + resendType := messagingtypes.ResendTypeRawMessage if chat.ChatType == ChatTypeOneToOne { - resendType = common.ResendTypeDataSync + resendType = messagingtypes.ResendTypeDataSync } - rawMessage, err := m.dispatchMessage(ctx, common.RawMessage{ + rawMessage, err := m.dispatchMessage(ctx, messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_REQUEST_TRANSACTION, @@ -4533,11 +4324,11 @@ func (m *Messenger) RequestAddressForTransaction(ctx context.Context, chatID, fr return nil, err } - resendType := common.ResendTypeRawMessage + resendType := messagingtypes.ResendTypeRawMessage if chat.ChatType == ChatTypeOneToOne { - resendType = common.ResendTypeDataSync + resendType = messagingtypes.ResendTypeDataSync } - rawMessage, err := m.dispatchMessage(ctx, common.RawMessage{ + rawMessage, err := m.dispatchMessage(ctx, messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_REQUEST_ADDRESS_FOR_TRANSACTION, @@ -4636,11 +4427,11 @@ func (m *Messenger) AcceptRequestAddressForTransaction(ctx context.Context, mess return nil, err } - resendType := common.ResendTypeRawMessage + resendType := messagingtypes.ResendTypeRawMessage if chat.ChatType == ChatTypeOneToOne { - resendType = common.ResendTypeDataSync + resendType = messagingtypes.ResendTypeDataSync } - rawMessage, err := m.dispatchMessage(ctx, common.RawMessage{ + rawMessage, err := m.dispatchMessage(ctx, messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_ACCEPT_REQUEST_ADDRESS_FOR_TRANSACTION, @@ -4720,11 +4511,11 @@ func (m *Messenger) DeclineRequestTransaction(ctx context.Context, messageID str return nil, err } - resendType := common.ResendTypeRawMessage + resendType := messagingtypes.ResendTypeRawMessage if chat.ChatType == ChatTypeOneToOne { - resendType = common.ResendTypeDataSync + resendType = messagingtypes.ResendTypeDataSync } - rawMessage, err := m.dispatchMessage(ctx, common.RawMessage{ + rawMessage, err := m.dispatchMessage(ctx, messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_DECLINE_REQUEST_TRANSACTION, @@ -4803,11 +4594,11 @@ func (m *Messenger) DeclineRequestAddressForTransaction(ctx context.Context, mes return nil, err } - resendType := common.ResendTypeRawMessage + resendType := messagingtypes.ResendTypeRawMessage if chat.ChatType == ChatTypeOneToOne { - resendType = common.ResendTypeDataSync + resendType = messagingtypes.ResendTypeDataSync } - rawMessage, err := m.dispatchMessage(ctx, common.RawMessage{ + rawMessage, err := m.dispatchMessage(ctx, messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_DECLINE_REQUEST_ADDRESS_FOR_TRANSACTION, @@ -4901,11 +4692,11 @@ func (m *Messenger) AcceptRequestTransaction(ctx context.Context, transactionHas return nil, err } - resendType := common.ResendTypeRawMessage + resendType := messagingtypes.ResendTypeRawMessage if chat.ChatType == ChatTypeOneToOne { - resendType = common.ResendTypeDataSync + resendType = messagingtypes.ResendTypeDataSync } - rawMessage, err := m.dispatchMessage(ctx, common.RawMessage{ + rawMessage, err := m.dispatchMessage(ctx, messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SEND_TRANSACTION, @@ -4979,11 +4770,11 @@ func (m *Messenger) SendTransaction(ctx context.Context, chatID, value, contract return nil, err } - resendType := common.ResendTypeRawMessage + resendType := messagingtypes.ResendTypeRawMessage if chat.ChatType == ChatTypeOneToOne { - resendType = common.ResendTypeDataSync + resendType = messagingtypes.ResendTypeDataSync } - rawMessage, err := m.dispatchMessage(ctx, common.RawMessage{ + rawMessage, err := m.dispatchMessage(ctx, messagingtypes.RawMessage{ LocalChatID: chat.ID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SEND_TRANSACTION, @@ -5338,27 +5129,6 @@ func (m *Messenger) GetPushNotificationsServers() ([]*pushnotificationclient.Pus return m.pushNotificationClient.GetServers() } -// StartPushNotificationsServer initialize and start a push notification server, using the current messenger identity key -func (m *Messenger) StartPushNotificationsServer() error { - if m.pushNotificationServer == nil { - pushNotificationServerPersistence := pushnotificationserver.NewSQLitePersistence(m.database) - config := &pushnotificationserver.Config{ - Enabled: true, - Logger: m.logger, - Identity: m.identity, - } - m.pushNotificationServer = pushnotificationserver.New(config, pushNotificationServerPersistence, m.sender) - } - - return m.pushNotificationServer.Start() -} - -// StopPushNotificationServer stops the push notification server if running -func (m *Messenger) StopPushNotificationsServer() error { - m.pushNotificationServer = nil - return nil -} - func generateAliasAndIdenticon(pk string) (string, string, error) { identicon, err := identicon.GenerateBase64(pk) if err != nil { @@ -5373,7 +5143,7 @@ func generateAliasAndIdenticon(pk string) (string, string, error) { } -func (m *Messenger) encodeChatEntity(chat *Chat, message common.ChatEntity) ([]byte, error) { +func (m *Messenger) encodeChatEntity(chat *Chat, message messagingtypes.ChatEntity) ([]byte, error) { var encodedMessage []byte var err error l := m.logger.With(zap.String("site", "Send"), zap.String("chatID", chat.ID)) @@ -5421,7 +5191,7 @@ func (m *Messenger) encodeChatEntity(chat *Chat, message common.ChatEntity) ([]b // NOTE(cammellos): Disabling for now since the optimiziation is not // applicable anymore after we changed group rules to allow // anyone to change group details - encodedMessage, err = m.sender.EncodeMembershipUpdate(group, message) + encodedMessage, err = encodeMembershipUpdate(group, message) if err != nil { return nil, err } @@ -5479,7 +5249,7 @@ func ToVerificationRequest(message *protobuf.SyncVerificationRequest) *verificat } } -func (m *Messenger) HandleSyncVerificationRequest(state *ReceivedMessageState, message *protobuf.SyncVerificationRequest, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncVerificationRequest(state *ReceivedMessageState, message *protobuf.SyncVerificationRequest, statusMessage *messagingtypes.Message) error { verificationRequest := ToVerificationRequest(message) err := m.verificationDatabase.SaveVerificationRequest(verificationRequest) @@ -5567,11 +5337,11 @@ func (m *Messenger) syncDeleteForMeMessage(ctx context.Context, rawMessageDispat if err2 != nil { return err2 } - rawMessage := common.RawMessage{ + rawMessage := messagingtypes.RawMessage{ LocalChatID: chatID, Payload: encodedMessage, MessageType: protobuf.ApplicationMetadataMessage_SYNC_DELETE_FOR_ME_MESSAGE, - ResendType: common.ResendTypeDataSync, + ResendType: messagingtypes.ResendTypeDataSync, } _, err2 = rawMessageDispatcher(ctx, rawMessage) if err2 != nil { @@ -5611,14 +5381,10 @@ func (m *Messenger) startCleanupLoop(name string, cleanupFunc func() error) { }() } -func (m *Messenger) startMessageSegmentsCleanupLoop() { - m.startCleanupLoop("messageSegmentsCleanupLoop", m.sender.CleanupSegments) -} - -func (m *Messenger) startHashRatchetEncryptedMessagesCleanupLoop() { - m.startCleanupLoop("hashRatchetEncryptedMessagesCleanupLoop", m.sender.CleanupHashRatchetEncryptedMessages) -} - func (m *Messenger) FindStatusMessageIDForBridgeMessageID(bridgeMessageID string) (string, error) { return m.persistence.FindStatusMessageIDForBridgeMessageID(bridgeMessageID) } + +func (m *Messenger) Messaging() *messaging.API { + return m.messaging +} diff --git a/protocol/messenger_backup.go b/protocol/messenger_backup.go deleted file mode 100644 index 263f28f35c9..00000000000 --- a/protocol/messenger_backup.go +++ /dev/null @@ -1,560 +0,0 @@ -package protocol - -import ( - "context" - "time" - - "github.com/golang/protobuf/proto" - "go.uber.org/zap" - - gocommon "github.com/status-im/status-go/common" - "github.com/status-im/status-go/multiaccounts/accounts" - multiaccountscommon "github.com/status-im/status-go/multiaccounts/common" - "github.com/status-im/status-go/multiaccounts/settings" - "github.com/status-im/status-go/protocol/common" - "github.com/status-im/status-go/protocol/communities" - "github.com/status-im/status-go/protocol/protobuf" -) - -const ( - BackupContactsPerBatch = 20 -) - -// backupTickerInterval is how often we should check for backups -var backupTickerInterval = 120 * time.Second - -// backupIntervalSeconds is the amount of seconds we should allow between -// backups -var backupIntervalSeconds uint64 = 28800 - -type CommunitySet struct { - Joined []*communities.Community - Deleted []*communities.Community -} - -func (m *Messenger) backupEnabled() (bool, error) { - return m.settings.BackupEnabled() -} - -func (m *Messenger) lastBackup() (uint64, error) { - return m.settings.LastBackup() -} - -func (m *Messenger) startBackupLoop() { - ticker := time.NewTicker(backupTickerInterval) - go func() { - defer gocommon.LogOnPanic() - for { - select { - case <-ticker.C: - if !m.Online() { - continue - } - - enabled, err := m.backupEnabled() - if err != nil { - m.logger.Error("failed to fetch backup enabled") - continue - } - if !enabled { - m.logger.Debug("backup not enabled, skipping") - continue - } - - lastBackup, err := m.lastBackup() - if err != nil { - m.logger.Error("failed to fetch last backup time") - continue - } - - now := time.Now().Unix() - if uint64(now) <= backupIntervalSeconds+lastBackup { - m.logger.Debug("not backing up") - continue - } - m.logger.Debug("backing up data") - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - _, err = m.BackupData(ctx) - if err != nil { - m.logger.Error("failed to backup data", zap.Error(err)) - } - case <-m.quit: - ticker.Stop() - return - } - } - }() -} - -func (m *Messenger) BackupData(ctx context.Context) (uint64, error) { - clock, chat := m.getLastClockWithRelatedChat() - contactsToBackup := m.backupContacts(ctx) - communitiesToBackup, err := m.backupCommunities(ctx, clock) - if err != nil { - return 0, err - } - chatsToBackup := m.backupChats(ctx, clock) - - profileToBackup, err := m.backupProfile(ctx, clock) - if err != nil { - return 0, err - } - _, settings, err := m.prepareSyncSettingsMessages(clock, true) - if err != nil { - // return just the first error, the others have been logged - return 0, err - } - - keypairsToBackup, err := m.backupKeypairs() - if err != nil { - return 0, err - } - - woAccountsToBackup, err := m.backupWatchOnlyAccounts() - if err != nil { - return 0, err - } - - backupDetailsOnly := func() *protobuf.Backup { - return &protobuf.Backup{ - Clock: clock, - ChatsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(len(chatsToBackup)), - }, - ContactsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(len(contactsToBackup)), - }, - CommunitiesDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(len(communitiesToBackup)), - }, - ProfileDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), // Profile is always one - }, - SettingsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(len(settings)), - }, - KeypairDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(len(keypairsToBackup)), - }, - WatchOnlyAccountDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(len(woAccountsToBackup)), - }, - } - } - - // Update contacts messages encode and dispatch - for i, d := range contactsToBackup { - pb := backupDetailsOnly() - pb.ContactsDetails.DataNumber = uint32(i + 1) - pb.Contacts = d.Contacts - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } - } - - // Update communities messages encode and dispatch - for i, d := range communitiesToBackup { - pb := backupDetailsOnly() - pb.CommunitiesDetails.DataNumber = uint32(i + 1) - pb.Communities = d.Communities - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } - } - - // Update profile messages encode and dispatch - pb := backupDetailsOnly() - pb.ProfileDetails.DataNumber = uint32(1) - pb.Profile = profileToBackup.Profile - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } - - // Update chats encode and dispatch - for i, d := range chatsToBackup { - pb := backupDetailsOnly() - pb.ChatsDetails.DataNumber = uint32(i + 1) - pb.Chats = d.Chats - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } - } - - // Update settings messages encode and dispatch - for i, d := range settings { - pb := backupDetailsOnly() - pb.SettingsDetails.DataNumber = uint32(i + 1) - pb.Setting = d - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } - } - - // TODO get rid of keypairs - // Update keypairs messages encode and dispatch - for i, d := range keypairsToBackup { - pb := backupDetailsOnly() - pb.KeypairDetails.DataNumber = uint32(i + 1) - pb.Keypair = d.Keypair - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } - } - - // Update watch only messages encode and dispatch - for i, d := range woAccountsToBackup { - pb := backupDetailsOnly() - pb.WatchOnlyAccountDetails.DataNumber = uint32(i + 1) - pb.WatchOnlyAccount = d.WatchOnlyAccount - err = m.encodeAndDispatchBackupMessage(ctx, pb, chat.ID) - if err != nil { - return 0, err - } - } - - chat.LastClockValue = clock - err = m.saveChat(chat) - if err != nil { - return 0, err - } - - clockInSeconds := clock / 1000 - err = m.settings.SetLastBackup(clockInSeconds) - if err != nil { - return 0, err - } - if m.config.messengerSignalsHandler != nil { - m.config.messengerSignalsHandler.BackupPerformed(clockInSeconds) - } - - return clockInSeconds, nil -} - -func (m *Messenger) encodeAndDispatchBackupMessage(ctx context.Context, message *protobuf.Backup, chatID string) error { - encodedMessage, err := proto.Marshal(message) - if err != nil { - return err - } - - _, err = m.dispatchMessage(ctx, common.RawMessage{ - LocalChatID: chatID, - Payload: encodedMessage, - SkipEncryptionLayer: true, - SendOnPersonalTopic: true, - MessageType: protobuf.ApplicationMetadataMessage_BACKUP, - }) - - return err -} - -func (m *Messenger) backupContacts(ctx context.Context) []*protobuf.Backup { - var contacts []*protobuf.SyncInstallationContactV2 - m.allContacts.Range(func(contactID string, contact *Contact) (shouldContinue bool) { - syncContact := m.buildSyncContactMessage(contact) - if syncContact != nil { - contacts = append(contacts, syncContact) - } - return true - }) - - var backupMessages []*protobuf.Backup - for i := 0; i < len(contacts); i += BackupContactsPerBatch { - j := i + BackupContactsPerBatch - if j > len(contacts) { - j = len(contacts) - } - - contactsToAdd := contacts[i:j] - - backupMessage := &protobuf.Backup{ - Contacts: contactsToAdd, - } - backupMessages = append(backupMessages, backupMessage) - } - - return backupMessages -} - -func (m *Messenger) retrieveAllCommunities() (*CommunitySet, error) { - joinedCs, err := m.communitiesManager.JoinedAndPendingCommunitiesWithRequests() - if err != nil { - return nil, err - } - - deletedCs, err := m.communitiesManager.DeletedCommunities() - if err != nil { - return nil, err - } - - return &CommunitySet{ - Joined: joinedCs, - Deleted: deletedCs, - }, nil -} - -func (m *Messenger) backupCommunities(ctx context.Context, clock uint64) ([]*protobuf.Backup, error) { - communitySet, err := m.retrieveAllCommunities() - if err != nil { - return nil, err - } - - var backupMessages []*protobuf.Backup - combinedCs := append(communitySet.Joined, communitySet.Deleted...) - - for _, c := range combinedCs { - _, beingImported := m.importingCommunities[c.IDString()] - if !beingImported { - backupMessage, err := m.backupCommunity(c, clock) - if err != nil { - return nil, err - } - - backupMessages = append(backupMessages, backupMessage) - } - } - - return backupMessages, nil -} - -func (m *Messenger) backupCommunity(community *communities.Community, clock uint64) (*protobuf.Backup, error) { - communityId := community.ID() - settings, err := m.communitiesManager.GetCommunitySettingsByID(communityId) - if err != nil { - return nil, err - } - - syncControlNode, err := m.communitiesManager.GetSyncControlNode(communityId) - if err != nil { - return nil, err - } - - syncMessage, err := community.ToSyncInstallationCommunityProtobuf(clock, settings, syncControlNode) - if err != nil { - return nil, err - } - - err = m.propagateSyncInstallationCommunityWithHRKeys(syncMessage, community) - if err != nil { - return nil, err - } - - return &protobuf.Backup{ - Communities: []*protobuf.SyncInstallationCommunity{syncMessage}, - }, nil -} - -func (m *Messenger) backupChats(ctx context.Context, clock uint64) []*protobuf.Backup { - var oneToOneAndGroupChats []*protobuf.SyncChat - m.allChats.Range(func(chatID string, chat *Chat) bool { - if !chat.OneToOne() && !chat.PrivateGroupChat() { - return true - } - syncChat := protobuf.SyncChat{ - Clock: clock, - Id: chatID, - ChatType: uint32(chat.ChatType), - Active: chat.Active, - } - chatMuteTill, _ := time.Parse(time.RFC3339, chat.MuteTill.Format(time.RFC3339)) - if chat.Muted && chatMuteTill.Equal(time.Time{}) { - // Only set Muted if it is "permanently" muted - syncChat.Muted = true - } - if chat.PrivateGroupChat() { - syncChat.Name = chat.Name // The Name is only useful in the case of a group chat - - syncChat.MembershipUpdateEvents = make([]*protobuf.MembershipUpdateEvents, len(chat.MembershipUpdates)) - for i, membershipUpdate := range chat.MembershipUpdates { - syncChat.MembershipUpdateEvents[i] = &protobuf.MembershipUpdateEvents{ - Clock: membershipUpdate.ClockValue, - Type: uint32(membershipUpdate.Type), - Members: membershipUpdate.Members, - Name: membershipUpdate.Name, - Signature: membershipUpdate.Signature, - ChatId: membershipUpdate.ChatID, - From: membershipUpdate.From, - RawPayload: membershipUpdate.RawPayload, - Color: membershipUpdate.Color, - Image: membershipUpdate.Image, - } - } - } - oneToOneAndGroupChats = append(oneToOneAndGroupChats, &syncChat) - return true - }) - - var backupMessages []*protobuf.Backup - backupMessage := &protobuf.Backup{ - Chats: oneToOneAndGroupChats, - } - backupMessages = append(backupMessages, backupMessage) - return backupMessages -} - -func (m *Messenger) buildSyncContactMessage(contact *Contact) *protobuf.SyncInstallationContactV2 { - var ensName string - if contact.ENSVerified { - ensName = contact.EnsName - } - - var customizationColor uint32 - if len(contact.CustomizationColor) != 0 { - customizationColor = multiaccountscommon.ColorToIDFallbackToBlue(contact.CustomizationColor) - } - - oneToOneChat, ok := m.allChats.Load(contact.ID) - muted := false - if ok { - muted = oneToOneChat.Muted - } - - return &protobuf.SyncInstallationContactV2{ - LastUpdatedLocally: contact.LastUpdatedLocally, - LastUpdated: contact.LastUpdated, - Id: contact.ID, - DisplayName: contact.DisplayName, - EnsName: ensName, - CustomizationColor: customizationColor, - LocalNickname: contact.LocalNickname, - Added: contact.added(), - Blocked: contact.Blocked, - Muted: muted, - HasAddedUs: contact.hasAddedUs(), - Removed: contact.Removed, - ContactRequestLocalState: int64(contact.ContactRequestLocalState), - ContactRequestRemoteState: int64(contact.ContactRequestRemoteState), - ContactRequestRemoteClock: int64(contact.ContactRequestRemoteClock), - ContactRequestLocalClock: int64(contact.ContactRequestLocalClock), - VerificationStatus: int64(contact.VerificationStatus), - TrustStatus: int64(contact.TrustStatus), - } -} - -func (m *Messenger) backupProfile(ctx context.Context, clock uint64) (*protobuf.Backup, error) { - displayName, err := m.settings.DisplayName() - if err != nil { - return nil, err - } - - displayNameClock, err := m.settings.GetSettingLastSynced(settings.DisplayName) - if err != nil { - return nil, err - } - - if m.account == nil { - return nil, nil - } - - keyUID := m.account.KeyUID - images, err := m.multiAccounts.GetIdentityImages(keyUID) - if err != nil { - return nil, err - } - - pictureProtos := make([]*protobuf.SyncProfilePicture, len(images)) - for i, image := range images { - p := &protobuf.SyncProfilePicture{} - p.Name = image.Name - p.Payload = image.Payload - p.Width = uint32(image.Width) - p.Height = uint32(image.Height) - p.FileSize = uint32(image.FileSize) - p.ResizeTarget = uint32(image.ResizeTarget) - if image.Clock == 0 { - p.Clock = clock - } else { - p.Clock = image.Clock - } - pictureProtos[i] = p - } - - ensUsernameDetails, err := m.getEnsUsernameDetails() - if err != nil { - return nil, err - } - ensUsernameDetailProtos := make([]*protobuf.SyncEnsUsernameDetail, len(ensUsernameDetails)) - for i, ensUsernameDetail := range ensUsernameDetails { - ensUsernameDetailProtos[i] = &protobuf.SyncEnsUsernameDetail{ - Username: ensUsernameDetail.Username, - Clock: ensUsernameDetail.Clock, - Removed: ensUsernameDetail.Removed, - ChainId: ensUsernameDetail.ChainID, - } - } - - profileShowcasePreferences, err := m.GetProfileShowcasePreferences() - if err != nil { - return nil, err - } - - backupMessage := &protobuf.Backup{ - Profile: &protobuf.BackedUpProfile{ - KeyUid: keyUID, - DisplayName: displayName, - Pictures: pictureProtos, - DisplayNameClock: displayNameClock, - EnsUsernameDetails: ensUsernameDetailProtos, - ProfileShowcasePreferences: ToProfileShowcasePreferencesProto(profileShowcasePreferences), - }, - } - - return backupMessage, nil -} - -func (m *Messenger) backupKeypairs() ([]*protobuf.Backup, error) { - keypairs, err := m.settings.GetAllKeypairs() - if err != nil { - return nil, err - } - - var backupMessages []*protobuf.Backup - for _, kp := range keypairs { - - kp.SyncedFrom = accounts.SyncedFromBackup - keypair, err := m.prepareSyncKeypairMessage(kp) - if err != nil { - return nil, err - } - - backupMessage := &protobuf.Backup{ - Keypair: keypair, - } - - backupMessages = append(backupMessages, backupMessage) - } - - return backupMessages, nil -} - -func (m *Messenger) backupWatchOnlyAccounts() ([]*protobuf.Backup, error) { - accounts, err := m.settings.GetAllWatchOnlyAccounts() - if err != nil { - return nil, err - } - - var backupMessages []*protobuf.Backup - for _, acc := range accounts { - - backupMessage := &protobuf.Backup{} - backupMessage.WatchOnlyAccount = m.prepareSyncAccountMessage(acc) - - backupMessages = append(backupMessages, backupMessage) - } - - return backupMessages, nil -} diff --git a/protocol/messenger_backup_handler.go b/protocol/messenger_backup_handler.go index 28fd0419179..60e0ccf62a3 100644 --- a/protocol/messenger_backup_handler.go +++ b/protocol/messenger_backup_handler.go @@ -2,22 +2,18 @@ package protocol import ( "database/sql" - "sync" - "time" "go.uber.org/zap" "github.com/golang/protobuf/proto" utils "github.com/status-im/status-go/common" - "github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/images" + messagingtypes "github.com/status-im/status-go/messaging/types" "github.com/status-im/status-go/multiaccounts/errors" "github.com/status-im/status-go/multiaccounts/settings" - "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" "github.com/status-im/status-go/protocol/protobuf" - v1protocol "github.com/status-im/status-go/protocol/v1" "github.com/status-im/status-go/protocol/wakusync" ensservice "github.com/status-im/status-go/services/ens" ) @@ -31,102 +27,6 @@ const ( SyncWakuSectionKeyWatchOnlyAccounts = "watchOnlyAccounts" ) -const backupSyncingNotificationID = "BACKUP_SYNCING" - -type FetchingBackedUpDataTracking struct { - LoadedItems map[uint32]bool - TotalNumber uint32 -} - -type BackupFetchingStatus struct { - dataProgress map[string]FetchingBackedUpDataTracking - lastKnownMsgClock uint64 - fetchingCompleted bool - fetchingCompletedMutex sync.Mutex -} - -func (m *Messenger) HandleBackup(state *ReceivedMessageState, message *protobuf.Backup, statusMessage *v1protocol.StatusMessage) error { - if !m.processBackedupMessages { - return nil - } - - errors := m.handleBackup(state, message) - if len(errors) > 0 { - for _, err := range errors { - m.logger.Warn("failed to handle Backup", zap.Error(err)) - } - return errors[0] - } - return nil -} - -// TODO remove this function once we do the Waku backup removal -func (m *Messenger) handleBackup(state *ReceivedMessageState, message *protobuf.Backup) []error { - var errors []error - - err := m.handleBackedUpProfile(message.Profile, message.Clock) - if err != nil { - errors = append(errors, err) - } - - for _, contact := range message.Contacts { - err = m.HandleSyncInstallationContactV2(state, contact, nil) - if err != nil { - errors = append(errors, err) - } - } - - err = m.handleSyncChats(state, message.Chats) - if err != nil { - errors = append(errors, err) - } - - communityErrors := m.handleSyncedCommunities(state, message) - if len(communityErrors) > 0 { - errors = append(errors, communityErrors...) - } - - err = m.HandleBackedUpSettings(message.Setting) - if err != nil { - errors = append(errors, err) - } - - err = m.handleKeypair(message.Keypair) - if err != nil { - errors = append(errors, err) - } - - err = m.HandleWatchOnlyAccount(message.WatchOnlyAccount) - if err != nil { - errors = append(errors, err) - } - - // Send signal about applied backup progress - if m.config.messengerSignalsHandler != nil { - response := wakusync.WakuBackedUpDataResponse{ - Clock: message.Clock, - } - - response.AddFetchingBackedUpDataDetails(SyncWakuSectionKeyProfile, message.ProfileDetails) - response.AddFetchingBackedUpDataDetails(SyncWakuSectionKeyContacts, message.ContactsDetails) - response.AddFetchingBackedUpDataDetails(SyncWakuSectionKeyCommunities, message.CommunitiesDetails) - response.AddFetchingBackedUpDataDetails(SyncWakuSectionKeySettings, message.SettingsDetails) - response.AddFetchingBackedUpDataDetails(SyncWakuSectionKeyKeypairs, message.KeypairDetails) - response.AddFetchingBackedUpDataDetails(SyncWakuSectionKeyWatchOnlyAccounts, message.WatchOnlyAccountDetails) - - err = m.updateBackupFetchProgress(message, &response, state) - if err != nil { - errors = append(errors, err) - } - - m.config.messengerSignalsHandler.SendWakuFetchingBackupProgress(&response) - } - - state.Response.BackupHandled = true - - return errors -} - func (m *Messenger) handleLocalBackup(state *ReceivedMessageState, message *protobuf.MessengerLocalBackup) []error { var errors []error @@ -155,154 +55,6 @@ func (m *Messenger) handleLocalBackup(state *ReceivedMessageState, message *prot return errors } -func (m *Messenger) updateBackupFetchProgress(message *protobuf.Backup, response *wakusync.WakuBackedUpDataResponse, state *ReceivedMessageState) error { - if m.backedUpFetchingStatus == nil { - return nil - } - - m.backedUpFetchingStatus.fetchingCompletedMutex.Lock() - defer m.backedUpFetchingStatus.fetchingCompletedMutex.Unlock() - - if m.backedUpFetchingStatus.fetchingCompleted { - return nil - } - - if m.backedUpFetchingStatus.lastKnownMsgClock > message.Clock { - return nil - } - - if m.backedUpFetchingStatus.lastKnownMsgClock < message.Clock { - // Reset the progress tracker because we have access to a more recent copy of the backup - m.backedUpFetchingStatus.lastKnownMsgClock = message.Clock - m.backedUpFetchingStatus.dataProgress = make(map[string]FetchingBackedUpDataTracking) - for backupName, details := range response.FetchingBackedUpDataDetails() { - m.backedUpFetchingStatus.dataProgress[backupName] = FetchingBackedUpDataTracking{ - LoadedItems: make(map[uint32]bool), - TotalNumber: details.TotalNumber, - } - } - - if len(m.backedUpFetchingStatus.dataProgress) == 0 { - return nil - } - } - - // Evaluate the progress of the backup - - // Set the new items before evaluating - for backupName, details := range response.FetchingBackedUpDataDetails() { - m.backedUpFetchingStatus.dataProgress[backupName].LoadedItems[details.DataNumber] = true - } - - for _, tracker := range m.backedUpFetchingStatus.dataProgress { - if len(tracker.LoadedItems)-1 < int(tracker.TotalNumber) { - // have not received everything yet - return nil - } - } - - m.backedUpFetchingStatus.fetchingCompleted = true - - // Update the AC notification and add it to the response - notification, err := m.persistence.GetActivityCenterNotificationByID(types.FromHex(backupSyncingNotificationID)) - if err != nil { - return err - } - - if notification == nil { - return nil - } - - notification.UpdatedAt = m.GetCurrentTimeInMillis() - notification.Type = ActivityCenterNotificationTypeBackupSyncingSuccess - _, err = m.persistence.SaveActivityCenterNotification(notification, true) - if err != nil { - return err - } - - state.Response.AddActivityCenterNotification(notification) - return nil -} - -func (m *Messenger) startBackupFetchingTracking(response *MessengerResponse) error { - // Add an acivity center notification to show that we are fetching back up messages - notification := &ActivityCenterNotification{ - ID: types.FromHex(backupSyncingNotificationID), - Type: ActivityCenterNotificationTypeBackupSyncingFetching, - Timestamp: m.getTimesource().GetCurrentTime(), - Read: false, - Deleted: false, - UpdatedAt: m.GetCurrentTimeInMillis(), - } - err := m.addActivityCenterNotification(response, notification, nil) - - if err != nil { - return err - } - - // Add a timeout to mark the backup syncing as failed after 1 minute and 30 seconds - m.shutdownWaitGroup.Add(1) - go func() { - defer utils.LogOnPanic() - defer m.shutdownWaitGroup.Done() - m.watchBackupFetching() - }() - - return nil -} - -func (m *Messenger) watchBackupFetching() { - select { - case <-m.ctx.Done(): - return - case <-time.After(90 * time.Second): - m.backupFetchingTimeout() - } -} - -func (m *Messenger) backupFetchingTimeout() { - if m.backedUpFetchingStatus == nil { - return - } - - m.backedUpFetchingStatus.fetchingCompletedMutex.Lock() - defer m.backedUpFetchingStatus.fetchingCompletedMutex.Unlock() - - if m.backedUpFetchingStatus.fetchingCompleted { - // Nothing to do, the fetching has completed successfully - return - } - // Update the AC notification to the failure state - notification, err := m.persistence.GetActivityCenterNotificationByID(types.FromHex(backupSyncingNotificationID)) - if err != nil { - m.logger.Error("failed to get activity center notification", zap.Error(err)) - return - } - if notification == nil { - return - } - - notification.UpdatedAt = m.GetCurrentTimeInMillis() - if m.backedUpFetchingStatus.dataProgress == nil || len(m.backedUpFetchingStatus.dataProgress) == 0 { - notification.Type = ActivityCenterNotificationTypeBackupSyncingFailure - } else { - notification.Type = ActivityCenterNotificationTypeBackupSyncingPartialFailure - } - _, err = m.persistence.SaveActivityCenterNotification(notification, true) - if err != nil { - m.logger.Error("failed to save activity center notification", zap.Error(err)) - return - } - - if m.config.messengerSignalsHandler == nil { - return - } - - resp := &MessengerResponse{} - resp.AddActivityCenterNotification(notification) - m.config.messengerSignalsHandler.MessengerResponse(resp) -} - func (m *Messenger) handleBackedUpProfile(message *protobuf.BackedUpProfile, backupTime uint64) error { if message == nil { return nil @@ -412,6 +164,7 @@ func (m *Messenger) handleBackedUpProfile(message *protobuf.BackedUpProfile, bac return err } +// TODO move this to the AccountsService func (m *Messenger) HandleBackedUpSettings(message *protobuf.SyncSetting) error { if message == nil { return nil @@ -458,64 +211,6 @@ func (m *Messenger) HandleBackedUpSettings(message *protobuf.SyncSetting) error return nil } -func (m *Messenger) handleKeypair(message *protobuf.SyncKeypair) error { - if message == nil { - return nil - } - - multiAcc, err := m.multiAccounts.GetAccount(message.KeyUid) - if err != nil { - return err - } - // If user is recovering his account via seed phrase, but the backed up messages indicate that the profile keypair - // is a keycard related profile, then we need to remove related profile keycards (only profile, other keycards should remain). - if multiAcc != nil && multiAcc.KeyUID == message.KeyUid && !multiAcc.RefersToKeycard() && len(message.Keycards) > 0 { - message.Keycards = []*protobuf.SyncKeycard{} - } - - keypair, err := m.handleSyncKeypair(message, false, nil) - if err != nil { - if err == ErrTryingToStoreOldKeypair { - return nil - } - return err - } - - if m.config.messengerSignalsHandler != nil { - kpResponse := wakusync.WakuBackedUpDataResponse{ - Keypair: keypair.CopyKeypair(), - } - - m.config.messengerSignalsHandler.SendWakuBackedUpKeypair(&kpResponse) - } - - return nil -} - -func (m *Messenger) HandleWatchOnlyAccount(message *protobuf.SyncAccount) error { - if message == nil { - return nil - } - - acc, err := m.handleSyncWatchOnlyAccount(message, true) - if err != nil { - if err == ErrTryingToStoreOldWalletAccount { - return nil - } - return err - } - - if m.config.messengerSignalsHandler != nil { - response := wakusync.WakuBackedUpDataResponse{ - WatchOnlyAccount: acc, - } - - m.config.messengerSignalsHandler.SendWakuBackedUpWatchOnlyAccount(&response) - } - - return nil -} - func syncInstallationCommunitiesSet(communities []*protobuf.SyncInstallationCommunity) map[string]*protobuf.SyncInstallationCommunity { ret := map[string]*protobuf.SyncInstallationCommunity{} for _, c := range communities { @@ -528,24 +223,6 @@ func syncInstallationCommunitiesSet(communities []*protobuf.SyncInstallationComm return ret } -// TODO remove this function once we do the Waku backup removal -func (m *Messenger) handleSyncedCommunities(state *ReceivedMessageState, message *protobuf.Backup) []error { - var errors []error - for _, syncCommunity := range syncInstallationCommunitiesSet(message.Communities) { - err := m.handleSyncInstallationCommunity(state, syncCommunity) - if err != nil { - errors = append(errors, err) - } - - err = m.requestCommunityKeysAndSharedAddresses(state, syncCommunity) - if err != nil { - errors = append(errors, err) - } - } - - return errors -} - func (m *Messenger) handleLocalBackupCommunities(state *ReceivedMessageState, communities []*protobuf.SyncInstallationCommunity) []error { var errors []error for _, syncCommunity := range syncInstallationCommunitiesSet(communities) { @@ -587,7 +264,7 @@ func (m *Messenger) requestCommunityKeysAndSharedAddresses(state *ReceivedMessag return err } - rawMessage := &common.RawMessage{ + rawMessage := &messagingtypes.RawMessage{ Payload: payload, Sender: m.identity, CommunityID: community.ID(), diff --git a/protocol/messenger_backup_test.go b/protocol/messenger_backup_test.go deleted file mode 100644 index 8bbef814228..00000000000 --- a/protocol/messenger_backup_test.go +++ /dev/null @@ -1,1203 +0,0 @@ -package protocol - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "testing" - "time" - - "go.uber.org/zap" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/event" - - v1protocol "github.com/status-im/status-go/protocol/v1" - "github.com/status-im/status-go/protocol/wakusync" - "github.com/status-im/status-go/services/accounts/accountsevent" - - "github.com/stretchr/testify/suite" - - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" - "github.com/status-im/status-go/images" - "github.com/status-im/status-go/multiaccounts/accounts" - "github.com/status-im/status-go/multiaccounts/settings" - "github.com/status-im/status-go/protocol/protobuf" - "github.com/status-im/status-go/protocol/requests" -) - -func TestMessengerBackupSuite(t *testing.T) { - suite.Run(t, new(MessengerBackupSuite)) -} - -type MessengerBackupSuite struct { - MessengerBaseTestSuite -} - -func (s *MessengerBackupSuite) TestBackupContacts() { - bob1 := s.m - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Create 2 contacts - contact1Key, err := crypto.GenerateKey() - s.Require().NoError(err) - contactID1 := types.EncodeHex(crypto.FromECDSAPub(&contact1Key.PublicKey)) - - _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID1}) - s.Require().NoError(err) - - contact2Key, err := crypto.GenerateKey() - s.Require().NoError(err) - contactID2 := types.EncodeHex(crypto.FromECDSAPub(&contact2Key.PublicKey)) - - _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID2}) - s.Require().NoError(err) - - s.Require().Len(bob1.Contacts(), 2) - - // Validate contacts - actualContacts := bob1.Contacts() - if actualContacts[0].ID == contactID1 { - s.Require().Equal(actualContacts[0].ID, contactID1) - s.Require().Equal(actualContacts[1].ID, contactID2) - } else { - s.Require().Equal(actualContacts[0].ID, contactID2) - s.Require().Equal(actualContacts[1].ID, contactID1) - } - - s.Require().Equal(ContactRequestStateSent, actualContacts[0].ContactRequestLocalState) - s.Require().Equal(ContactRequestStateSent, actualContacts[1].ContactRequestLocalState) - s.Require().True(actualContacts[0].added()) - s.Require().True(actualContacts[1].added()) - - // Backup - clock, err := bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Safety check - s.Require().Len(bob2.Contacts(), 0) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - s.Require().Len(bob2.AddedContacts(), 2) - - actualContacts = bob2.AddedContacts() - if actualContacts[0].ID == contactID1 { - s.Require().Equal(actualContacts[0].ID, contactID1) - s.Require().Equal(actualContacts[1].ID, contactID2) - } else { - s.Require().Equal(actualContacts[0].ID, contactID2) - s.Require().Equal(actualContacts[1].ID, contactID1) - } - s.Require().Equal(ContactRequestStateSent, actualContacts[0].ContactRequestLocalState) - s.Require().Equal(ContactRequestStateSent, actualContacts[1].ContactRequestLocalState) - lastBackup, err := bob1.lastBackup() - s.Require().NoError(err) - s.Require().NotEmpty(lastBackup) - s.Require().Equal(clock, lastBackup) -} - -func (s *MessengerBackupSuite) TestBackupProfile() { - const bob1DisplayName = "bobby" - - // Create bob1 - bob1 := s.m - - bobProfileKp := accounts.GetProfileKeypairForTest(true, false, false) - bobProfileKp.KeyUID = bob1.account.KeyUID - bobProfileKp.Accounts[0].KeyUID = bob1.account.KeyUID - - err := bob1.settings.SaveOrUpdateKeypair(bobProfileKp) - s.Require().NoError(err) - - err = bob1.SetDisplayName(bob1DisplayName) - s.Require().NoError(err) - bob1KeyUID := bob1.account.KeyUID - imagesExpected := fmt.Sprintf(`[{"keyUid":"%s","type":"large","uri":"data:image/png;base64,iVBORw0KGgoAAAANSUg=","width":240,"height":300,"fileSize":1024,"resizeTarget":240,"clock":0},{"keyUid":"%s","type":"thumbnail","uri":"data:image/jpeg;base64,/9j/2wCEAFA3PEY8MlA=","width":80,"height":80,"fileSize":256,"resizeTarget":80,"clock":0}]`, - bob1KeyUID, bob1KeyUID) - - iis := images.SampleIdentityImages() - s.Require().NoError(bob1.multiAccounts.StoreIdentityImages(bob1KeyUID, iis, false)) - - bob1EnsUsernameDetail, err := bob1.saveEnsUsernameDetailProto(&protobuf.SyncEnsUsernameDetail{ - Clock: 1, - Username: "bob1.eth", - ChainId: 1, - Removed: false, - }) - s.Require().NoError(err) - - profileShowcasePreferences := DummyProfileShowcasePreferences(false) - err = bob1.SetProfileShowcasePreferences(profileShowcasePreferences, false) - s.Require().NoError(err) - - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Check bob1 - storedBob1DisplayName, err := bob1.settings.DisplayName() - s.Require().NoError(err) - s.Require().Equal(bob1DisplayName, storedBob1DisplayName) - - bob1Images, err := bob1.multiAccounts.GetIdentityImages(bob1KeyUID) - s.Require().NoError(err) - jBob1Images, err := json.Marshal(bob1Images) - s.Require().NoError(err) - s.Require().Equal(imagesExpected, string(jBob1Images)) - - bob1EnsUsernameDetails, err := bob1.getEnsUsernameDetails() - s.Require().NoError(err) - s.Require().Equal(1, len(bob1EnsUsernameDetails)) - - bob1ProfileShowcasePreferences, err := bob1.GetProfileShowcasePreferences() - s.Require().NoError(err) - s.Require().NotNil(bob1ProfileShowcasePreferences) - s.Require().Greater(bob1ProfileShowcasePreferences.Clock, uint64(0)) - profileShowcasePreferences.Clock = bob1ProfileShowcasePreferences.Clock // override clock for simpler comparison - s.Require().True(reflect.DeepEqual(profileShowcasePreferences, bob1ProfileShowcasePreferences)) - - // Check bob2 - storedBob2DisplayName, err := bob2.settings.DisplayName() - s.Require().NoError(err) - s.Require().Equal(DefaultProfileDisplayName, storedBob2DisplayName) - - var expectedEmpty []*images.IdentityImage - bob2Images, err := bob2.multiAccounts.GetIdentityImages(bob1KeyUID) - s.Require().NoError(err) - s.Require().Equal(expectedEmpty, bob2Images) - - bob2EnsUsernameDetails, err := bob2.getEnsUsernameDetails() - s.Require().NoError(err) - s.Require().Equal(0, len(bob2EnsUsernameDetails)) - - bob2ProfileShowcasePreferences, err := bob2.GetProfileShowcasePreferences() - s.Require().NoError(err) - s.Require().NotNil(bob2ProfileShowcasePreferences) - s.Require().Equal(uint64(0), bob2ProfileShowcasePreferences.Clock) - s.Require().Len(bob2ProfileShowcasePreferences.Communities, 0) - s.Require().Len(bob2ProfileShowcasePreferences.Accounts, 0) - s.Require().Len(bob2ProfileShowcasePreferences.Collectibles, 0) - s.Require().Len(bob2ProfileShowcasePreferences.VerifiedTokens, 0) - s.Require().Len(bob2ProfileShowcasePreferences.UnverifiedTokens, 0) - s.Require().Len(bob2ProfileShowcasePreferences.SocialLinks, 0) - - // Backup - clock, err := bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - - // Check bob2 - storedBob2DisplayName, err = bob2.settings.DisplayName() - s.Require().NoError(err) - s.Require().Equal(bob1DisplayName, storedBob2DisplayName) - - bob2Images, err = bob2.multiAccounts.GetIdentityImages(bob1KeyUID) - s.Require().NoError(err) - s.Require().Equal(2, len(bob2Images)) - s.Require().Equal(bob2Images[0].Payload, bob1Images[0].Payload) - s.Require().Equal(bob2Images[1].Payload, bob1Images[1].Payload) - - bob2EnsUsernameDetails, err = bob2.getEnsUsernameDetails() - s.Require().NoError(err) - s.Require().Equal(1, len(bob2EnsUsernameDetails)) - s.Require().Equal(bob1EnsUsernameDetail, bob2EnsUsernameDetails[0]) - - bob2ProfileShowcasePreferences, err = bob1.GetProfileShowcasePreferences() - s.Require().NoError(err) - s.Require().True(reflect.DeepEqual(bob1ProfileShowcasePreferences, bob2ProfileShowcasePreferences)) - - lastBackup, err := bob1.lastBackup() - s.Require().NoError(err) - s.Require().NotEmpty(lastBackup) - s.Require().Equal(clock, lastBackup) -} - -func (s *MessengerBackupSuite) TestBackupProfileWithInvalidDisplayName() { - // Create bob1 - bob1 := s.m - - state := ReceivedMessageState{ - Response: &MessengerResponse{}, - } - - err := bob1.HandleBackup( - &state, - &protobuf.Backup{ - Profile: &protobuf.BackedUpProfile{ - DisplayName: "bad-display-name_eth", - }, - Clock: 1, - }, - &v1protocol.StatusMessage{}, - ) - // The backup will still work, but the display name will be skipped - s.Require().NoError(err) - storedBob1DisplayName, err := bob1.settings.DisplayName() - s.Require().NoError(err) - s.Require().Equal("", storedBob1DisplayName) -} - -func (s *MessengerBackupSuite) TestFetchingDuringBackup() { - bob1 := s.m - bob1.config.messengerSignalsHandler = &MessengerSignalsHandlerMock{ - wakuBackedUpDataResponseChan: make(chan *wakusync.WakuBackedUpDataResponse, 1000), - } - - state := ReceivedMessageState{ - Response: &MessengerResponse{}, - } - - backup := &protobuf.Backup{ - Clock: 1, - ContactsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - CommunitiesDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - ProfileDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - } - - err := bob1.HandleBackup( - &state, - backup, - &v1protocol.StatusMessage{}, - ) - s.Require().NoError(err) - // The backup is not done, so no signal should be sent - s.Require().Len(state.Response.ActivityCenterNotifications(), 0) - s.Require().Len(bob1.backedUpFetchingStatus.dataProgress, 3) - s.Require().Equal(uint32(1), bob1.backedUpFetchingStatus.dataProgress[SyncWakuSectionKeyContacts].TotalNumber) - - // Parse a backup with a higher clock so reset the fetching - backup = &protobuf.Backup{ - Clock: 2, - ContactsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(2), - }, - CommunitiesDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - ProfileDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - SettingsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - KeypairDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - WatchOnlyAccountDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - } - err = bob1.HandleBackup( - &state, - backup, - &v1protocol.StatusMessage{}, - ) - s.Require().NoError(err) - // The backup is not done, so no signal should be sent - s.Require().Len(state.Response.ActivityCenterNotifications(), 0) - s.Require().Len(bob1.backedUpFetchingStatus.dataProgress, 6) - s.Require().Equal(uint32(2), bob1.backedUpFetchingStatus.dataProgress[SyncWakuSectionKeyContacts].TotalNumber) - - // Backup with a smaller clock is ignored - backup = &protobuf.Backup{ - Clock: 2, - ContactsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(5), - }, - CommunitiesDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - } - err = bob1.HandleBackup( - &state, - backup, - &v1protocol.StatusMessage{}, - ) - s.Require().NoError(err) - // The backup is not done, so no signal should be sent - s.Require().Len(state.Response.ActivityCenterNotifications(), 0) - // The values are gonna be the same as before as the backup was ignored - s.Require().Len(bob1.backedUpFetchingStatus.dataProgress, 6) - s.Require().Equal(uint32(2), bob1.backedUpFetchingStatus.dataProgress[SyncWakuSectionKeyContacts].TotalNumber) - - // Parse the backup with almost all the correct data numbers - backup = &protobuf.Backup{ - Clock: 2, - ContactsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(1), - TotalNumber: uint32(2), - }, - CommunitiesDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(1), - TotalNumber: uint32(1), - }, - ProfileDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(1), - TotalNumber: uint32(1), - }, - SettingsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(1), - TotalNumber: uint32(1), - }, - KeypairDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(1), - TotalNumber: uint32(1), - }, - WatchOnlyAccountDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(1), - TotalNumber: uint32(1), - }, - } - err = bob1.HandleBackup( - &state, - backup, - &v1protocol.StatusMessage{}, - ) - s.Require().NoError(err) - // The backup is not done, so no signal should be sent - s.Require().Len(state.Response.ActivityCenterNotifications(), 0) - - // Parse the remaining backup so the notification should be sent now - backup = &protobuf.Backup{ - Clock: 2, - ContactsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(2), - TotalNumber: uint32(2), - }, - } - err = bob1.HandleBackup( - &state, - backup, - &v1protocol.StatusMessage{}, - ) - s.Require().NoError(err) - // The backup is done, so the signal should be sent - s.Require().Len(state.Response.ActivityCenterNotifications(), 1) - s.Require().Equal(ActivityCenterNotificationTypeBackupSyncingSuccess, state.Response.ActivityCenterNotifications()[0].Type) -} - -func (s *MessengerBackupSuite) TestBackupWithoutStatusFetching() { - bob1 := s.m - // Remove the status fetching - bob1.backedUpFetchingStatus = nil - bob1.config.messengerSignalsHandler = &MessengerSignalsHandlerMock{ - wakuBackedUpDataResponseChan: make(chan *wakusync.WakuBackedUpDataResponse, 1000), - } - - state := ReceivedMessageState{ - Response: &MessengerResponse{}, - } - - backup := &protobuf.Backup{ - Clock: 1, - ContactsDetails: &protobuf.FetchingBackedUpDataDetails{ - DataNumber: uint32(0), - TotalNumber: uint32(1), - }, - } - - err := bob1.HandleBackup( - &state, - backup, - &v1protocol.StatusMessage{}, - ) - s.Require().NoError(err) - // The backup was handled but nothing was sent to the AC - s.Require().Len(state.Response.ActivityCenterNotifications(), 0) - s.Require().True(state.Response.BackupHandled) -} - -func (s *MessengerBackupSuite) TestBackupSettings() { - s.T().Skip("flaky test") - const ( - bob1DisplayName = "bobby" - bob1Currency = "eur" - bob1MessagesFromContactsOnly = true - bob1ProfilePicturesShowTo = settings.ProfilePicturesShowToEveryone - bob1ProfilePicturesVisibility = settings.ProfilePicturesVisibilityEveryone - bob1Bio = "bio" - bob1Mnemonic = "" - bob1MnemonicRemoved = true - bob1PreferredName = "talent" - bob1UrlUnfUnfurlingMode = settings.URLUnfurlingEnableAll - ) - - // Create bob1 and set fields which are supposed to be backed up to/fetched from waku - bob1 := s.m - err := bob1.settings.SaveSettingField(settings.DisplayName, bob1DisplayName) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.Currency, bob1Currency) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.MessagesFromContactsOnly, bob1MessagesFromContactsOnly) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.ProfilePicturesShowTo, bob1ProfilePicturesShowTo) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.ProfilePicturesVisibility, bob1ProfilePicturesVisibility) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.Bio, bob1Bio) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.Mnemonic, bob1Mnemonic) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.PreferredName, bob1PreferredName) - s.Require().NoError(err) - err = bob1.settings.SaveSettingField(settings.URLUnfurlingMode, bob1UrlUnfUnfurlingMode) - s.Require().NoError(err) - - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Check bob1 - storedBob1DisplayName, err := bob1.settings.DisplayName() - s.Require().NoError(err) - s.Require().Equal(bob1DisplayName, storedBob1DisplayName) - storedBob1Currency, err := bob1.settings.GetCurrency() - s.Require().NoError(err) - s.Require().Equal(bob1Currency, storedBob1Currency) - storedBob1MessagesFromContactsOnly, err := bob1.settings.GetMessagesFromContactsOnly() - s.Require().NoError(err) - s.Require().Equal(bob1MessagesFromContactsOnly, storedBob1MessagesFromContactsOnly) - storedBob1ProfilePicturesShowTo, err := bob1.settings.GetProfilePicturesShowTo() - s.Require().NoError(err) - s.Require().Equal(int64(bob1ProfilePicturesShowTo), storedBob1ProfilePicturesShowTo) - storedBob1ProfilePicturesVisibility, err := bob1.settings.GetProfilePicturesVisibility() - s.Require().NoError(err) - s.Require().Equal(int(bob1ProfilePicturesVisibility), storedBob1ProfilePicturesVisibility) - storedBob1Bio, err := bob1.settings.Bio() - s.Require().NoError(err) - s.Require().Equal(bob1Bio, storedBob1Bio) - storedMnemonic, err := bob1.settings.Mnemonic() - s.Require().NoError(err) - s.Require().Equal(bob1Mnemonic, storedMnemonic) - storedMnemonicRemoved, err := bob1.settings.MnemonicRemoved() - s.Require().NoError(err) - s.Require().Equal(bob1MnemonicRemoved, storedMnemonicRemoved) - storedPreferredName, err := bob1.settings.GetPreferredUsername() - s.NoError(err) - s.Require().Equal(bob1PreferredName, storedPreferredName) - storedBob1UrlUnfurlingMode, err := bob1.settings.URLUnfurlingMode() - s.NoError(err) - s.Require().Equal(int64(bob1UrlUnfUnfurlingMode), storedBob1UrlUnfurlingMode) - - // Check bob2 - storedBob2DisplayName, err := bob2.settings.DisplayName() - s.Require().NoError(err) - s.Require().NotEqual(storedBob1DisplayName, storedBob2DisplayName) - storedBob2Currency, err := bob2.settings.GetCurrency() - s.Require().NoError(err) - s.Require().NotEqual(storedBob1Currency, storedBob2Currency) - storedBob2MessagesFromContactsOnly, err := bob2.settings.GetMessagesFromContactsOnly() - s.Require().NoError(err) - s.Require().NotEqual(storedBob1MessagesFromContactsOnly, storedBob2MessagesFromContactsOnly) - storedBob2ProfilePicturesShowTo, err := bob2.settings.GetProfilePicturesShowTo() - s.Require().NoError(err) - s.Require().NotEqual(storedBob1ProfilePicturesShowTo, storedBob2ProfilePicturesShowTo) - storedBob2ProfilePicturesVisibility, err := bob2.settings.GetProfilePicturesVisibility() - s.Require().NoError(err) - s.Require().NotEqual(storedBob1ProfilePicturesVisibility, storedBob2ProfilePicturesVisibility) - storedBob2Bio, err := bob2.settings.Bio() - s.Require().NoError(err) - s.Require().NotEqual(storedBob1Bio, storedBob2Bio) - storedBob2MnemonicRemoved, err := bob2.settings.MnemonicRemoved() - s.Require().NoError(err) - s.Require().Equal(false, storedBob2MnemonicRemoved) - storedBob2PreferredName, err := bob2.settings.GetPreferredUsername() - s.NoError(err) - s.Require().Equal("", storedBob2PreferredName) - storedBob2UrlUnfurlingMode, err := bob2.settings.URLUnfurlingMode() - s.NoError(err) - s.Require().Equal(int64(settings.URLUnfurlingAlwaysAsk), storedBob2UrlUnfurlingMode) - - // Backup - clock, err := bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Wait for the message to reach its destination - _, err = WaitOnSignaledSendWakuFetchingBackupProgress(bob2, func(r *wakusync.WakuBackedUpDataResponse) bool { - detailsMap := r.FetchingBackedUpDataDetails() - if settingDetails, ok := detailsMap[SyncWakuSectionKeySettings]; ok { - return settingDetails.DataNumber == settingDetails.TotalNumber - } - return false - }, "no messages") - s.Require().NoError(err) - - // Check bob2 - storedBob2DisplayName, err = bob2.settings.DisplayName() - s.Require().NoError(err) - s.Require().Equal(storedBob1DisplayName, storedBob2DisplayName) - storedBob2Currency, err = bob2.settings.GetCurrency() - s.Require().NoError(err) - s.Require().Equal(storedBob1Currency, storedBob2Currency) - storedBob2MessagesFromContactsOnly, err = bob2.settings.GetMessagesFromContactsOnly() - s.Require().NoError(err) - s.Require().Equal(storedBob1MessagesFromContactsOnly, storedBob2MessagesFromContactsOnly) - storedBob2ProfilePicturesShowTo, err = bob2.settings.GetProfilePicturesShowTo() - s.Require().NoError(err) - s.Require().Equal(storedBob1ProfilePicturesShowTo, storedBob2ProfilePicturesShowTo) - storedBob2ProfilePicturesVisibility, err = bob2.settings.GetProfilePicturesVisibility() - s.Require().NoError(err) - s.Require().Equal(storedBob1ProfilePicturesVisibility, storedBob2ProfilePicturesVisibility) - storedBob2Bio, err = bob2.settings.Bio() - s.Require().NoError(err) - s.Require().Equal(storedBob1Bio, storedBob2Bio) - storedBob2MnemonicRemoved, err = bob2.settings.MnemonicRemoved() - s.Require().NoError(err) - s.Require().Equal(bob1MnemonicRemoved, storedBob2MnemonicRemoved) - storedBob2PreferredName, err = bob2.settings.GetPreferredUsername() - s.NoError(err) - s.Require().Equal(bob1PreferredName, storedBob2PreferredName) - s.Require().Equal(bob1PreferredName, bob2.account.Name) - storedBob2UrlUnfurlingMode, err = bob2.settings.URLUnfurlingMode() - s.NoError(err) - s.Require().Equal(storedBob1UrlUnfurlingMode, storedBob2UrlUnfurlingMode) - - lastBackup, err := bob1.lastBackup() - s.Require().NoError(err) - s.Require().NotEmpty(lastBackup) - s.Require().Equal(clock, lastBackup) -} - -func (s *MessengerBackupSuite) TestBackupContactsGreaterThanBatch() { - bob1 := s.m - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Create contacts - - for i := 0; i < BackupContactsPerBatch*2; i++ { - - contactKey, err := crypto.GenerateKey() - s.Require().NoError(err) - contactID := types.EncodeHex(crypto.FromECDSAPub(&contactKey.PublicKey)) - - _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID}) - s.Require().NoError(err) - - } - // Backup - - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Safety check - s.Require().Len(bob2.Contacts(), 0) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - - s.Require().Equal(BackupContactsPerBatch*2, len(bob2.Contacts())) - s.Require().Len(bob2.AddedContacts(), BackupContactsPerBatch*2) -} - -func (s *MessengerBackupSuite) TestBackupRemovedContact() { - bob1 := s.m - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Create 2 contacts on bob 1 - - contact1Key, err := crypto.GenerateKey() - s.Require().NoError(err) - contactID1 := types.EncodeHex(crypto.FromECDSAPub(&contact1Key.PublicKey)) - - _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID1}) - s.Require().NoError(err) - - contact2Key, err := crypto.GenerateKey() - s.Require().NoError(err) - contactID2 := types.EncodeHex(crypto.FromECDSAPub(&contact2Key.PublicKey)) - - _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID2}) - s.Require().NoError(err) - - s.Require().Len(bob1.Contacts(), 2) - - actualContacts := bob1.Contacts() - if actualContacts[0].ID == contactID1 { - s.Require().Equal(actualContacts[0].ID, contactID1) - s.Require().Equal(actualContacts[1].ID, contactID2) - } else { - s.Require().Equal(actualContacts[0].ID, contactID2) - s.Require().Equal(actualContacts[1].ID, contactID1) - } - - // Bob 2 add one of the same contacts - - _, err = bob2.AddContact(context.Background(), &requests.AddContact{ID: contactID2}) - s.Require().NoError(err) - - // Bob 1 now removes one of the contact that was also on bob 2 - - _, err = bob1.RemoveContact(context.Background(), contactID2) - s.Require().NoError(err) - - // Backup - - clock, err := bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Safety check - s.Require().Len(bob2.Contacts(), 1) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - // Bob 2 should remove the contact - s.Require().NoError(err) - - s.Require().Len(bob2.AddedContacts(), 1) - s.Require().Equal(contactID1, bob2.AddedContacts()[0].ID) - - lastBackup, err := bob1.lastBackup() - s.Require().NoError(err) - s.Require().NotEmpty(lastBackup) - s.Require().Equal(clock, lastBackup) -} - -func (s *MessengerBackupSuite) TestBackupLocalNickname() { - bob1 := s.m - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - nickname := "don van vliet" - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Set contact nickname - - contact1Key, err := crypto.GenerateKey() - s.Require().NoError(err) - contactID1 := types.EncodeHex(crypto.FromECDSAPub(&contact1Key.PublicKey)) - - _, err = bob1.SetContactLocalNickname(&requests.SetContactLocalNickname{ID: types.Hex2Bytes(contactID1), Nickname: nickname}) - s.Require().NoError(err) - - // Backup - - clock, err := bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Safety check - s.Require().Len(bob2.Contacts(), 0) - - var actualContact *Contact - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - - for _, c := range bob2.Contacts() { - if c.ID == contactID1 { - actualContact = c - break - } - } - - s.Require().Equal(actualContact.LocalNickname, nickname) - lastBackup, err := bob1.lastBackup() - s.Require().NoError(err) - s.Require().NotEmpty(lastBackup) - s.Require().Equal(clock, lastBackup) -} - -func (s *MessengerBackupSuite) TestBackupBlockedContacts() { - bob1 := s.m - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Create contact - contact1Key, err := crypto.GenerateKey() - s.Require().NoError(err) - contactID1 := types.EncodeHex(crypto.FromECDSAPub(&contact1Key.PublicKey)) - - _, err = bob1.AddContact(context.Background(), &requests.AddContact{ID: contactID1}) - s.Require().NoError(err) - - // Backup - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Safety check - s.Require().Len(bob2.Contacts(), 0) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - - s.Require().Len(bob2.AddedContacts(), 1) - - actualContacts := bob2.AddedContacts() - s.Require().Equal(actualContacts[0].ID, contactID1) - - _, err = bob1.BlockContact(context.Background(), contactID1, false) - s.Require().NoError(err) - - // Backup - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - s.Require().Len(bob2.BlockedContacts(), 1) -} - -func (s *MessengerBackupSuite) TestBackupCommunities() { - bob1 := s.m - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Create a community - description := &requests.CreateCommunity{ - Membership: protobuf.CommunityPermissions_AUTO_ACCEPT, - Name: "status", - Color: "#ffffff", - Description: "status community description", - } - - // Create a community chat - response, err := bob1.CreateCommunity(description, true) - s.Require().NoError(err) - s.Require().NotNil(response) - s.Require().Len(response.Communities(), 1) - - // Backup - clock, err := bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Safety check - communities, err := bob2.Communities() - s.Require().NoError(err) - s.Require().Len(communities, 0) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - - s.Require().NoError(err) - - communities, err = bob2.JoinedCommunities() - s.Require().NoError(err) - s.Require().Len(communities, 1) - - lastBackup, err := bob1.lastBackup() - s.Require().NoError(err) - s.Require().NotEmpty(lastBackup) - s.Require().Equal(clock, lastBackup) -} - -func (s *MessengerBackupSuite) TestBackupKeypairs() { - // Create bob1 - bob1 := s.m - profileKp := accounts.GetProfileKeypairForTest(true, true, true) - seedKp := accounts.GetSeedImportedKeypair1ForTest() - - // Create a main account on bob1 - err := bob1.settings.SaveOrUpdateKeypair(profileKp) - s.Require().NoError(err, "profile keypair bob1") - err = bob1.settings.SaveOrUpdateKeypair(seedKp) - s.Require().NoError(err, "seed keypair bob1") - - // Check account is present in the db for bob1 - dbProfileKp1, err := bob1.settings.GetKeypairByKeyUID(profileKp.KeyUID) - s.Require().NoError(err) - s.Require().True(accounts.SameKeypairs(profileKp, dbProfileKp1)) - dbSeedKp1, err := bob1.settings.GetKeypairByKeyUID(seedKp.KeyUID) - s.Require().NoError(err) - s.Require().True(accounts.SameKeypairs(seedKp, dbSeedKp1)) - - // Create bob2 - accountsFeed := &event.Feed{} - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, []Option{WithAccountsFeed(accountsFeed)}) - s.Require().NoError(err) - s.Require().NotNil(bob2.config.accountsFeed) - ch := make(chan accountsevent.Event, 20) - sub := bob2.config.accountsFeed.Subscribe(ch) - defer TearDownMessenger(&s.Suite, bob2) - - // Backup - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - - // Check account is present in the db for bob2 - dbProfileKp2, err := bob2.settings.GetKeypairByKeyUID(profileKp.KeyUID) - s.Require().NoError(err) - s.Require().Equal(profileKp.Name, dbProfileKp2.Name) - s.Require().Equal(accounts.SyncedFromBackup, dbProfileKp2.SyncedFrom) - - for _, acc := range profileKp.Accounts { - s.Require().True(accounts.Contains(dbProfileKp2.Accounts, acc, accounts.SameAccounts)) - } - - dbSeedKp2, err := bob2.settings.GetKeypairByKeyUID(seedKp.KeyUID) - s.Require().NoError(err) - s.Require().True(accounts.SameKeypairsWithDifferentSyncedFrom(seedKp, dbSeedKp2, false, accounts.SyncedFromBackup, accounts.AccountNonOperable)) - - keypairs, err := bob2.settings.GetAllKeypairs() - s.Require().NoError(err) - - // Check whether accounts added event is sent - expectedAddresses := make(map[common.Address]struct{}, 0) - for _, acc := range dbProfileKp2.Accounts { - if acc.Chat { - continue - } - expectedAddresses[common.Address(acc.Address)] = struct{}{} - } - - for _, acc := range dbSeedKp2.Accounts { - expectedAddresses[common.Address(acc.Address)] = struct{}{} - } - - for i := 0; i < len(keypairs); i++ { - select { - case <-time.After(1 * time.Second): - s.Fail("Timed out waiting for accountsevent") - case event := <-ch: - switch event.Type { - case accountsevent.EventTypeAdded: - for _, address := range event.Accounts { - if _, exists := expectedAddresses[address]; !exists { - s.logger.Debug("missing address in the accounts event", zap.Any("address", address)) - s.Fail("address not received in the event") - } - } - } - } - } - sub.Unsubscribe() -} - -func (s *MessengerBackupSuite) TestBackupKeycards() { - // Create bob1 - bob1 := s.m - - kp1 := accounts.GetProfileKeypairForTest(true, true, true) - keycard1 := accounts.GetProfileKeycardForTest() - - kp2 := accounts.GetSeedImportedKeypair1ForTest() - keycard2 := accounts.GetKeycardForSeedImportedKeypair1ForTest() - - keycard2Copy := accounts.GetKeycardForSeedImportedKeypair1ForTest() - keycard2Copy.KeycardUID = keycard2Copy.KeycardUID + "C" - keycard2Copy.KeycardName = keycard2Copy.KeycardName + "Copy" - - kp3 := accounts.GetSeedImportedKeypair2ForTest() - keycard3 := accounts.GetKeycardForSeedImportedKeypair2ForTest() - - // Pre-condition - err := bob1.settings.SaveOrUpdateKeypair(kp1) - s.Require().NoError(err) - err = bob1.settings.SaveOrUpdateKeypair(kp2) - s.Require().NoError(err) - err = bob1.settings.SaveOrUpdateKeypair(kp3) - s.Require().NoError(err) - dbKeypairs, err := bob1.settings.GetActiveKeypairs() - s.Require().NoError(err) - s.Require().Equal(3, len(dbKeypairs)) - - err = bob1.settings.SaveOrUpdateKeycard(*keycard1, 0, false) - s.Require().NoError(err) - err = bob1.settings.SaveOrUpdateKeycard(*keycard2, 0, false) - s.Require().NoError(err) - err = bob1.settings.SaveOrUpdateKeycard(*keycard2Copy, 0, false) - s.Require().NoError(err) - err = bob1.settings.SaveOrUpdateKeycard(*keycard3, 0, false) - s.Require().NoError(err) - - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Backup - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - syncedKeycards, err := bob2.settings.GetAllKnownKeycards() - s.Require().NoError(err) - return r.BackupHandled && len(syncedKeycards) == 4 - - }, - "no messages", - ) - s.Require().NoError(err) - - syncedKeycards, err := bob2.settings.GetAllKnownKeycards() - s.Require().NoError(err) - s.Require().Equal(4, len(syncedKeycards)) - s.Require().True(accounts.Contains(syncedKeycards, keycard1, accounts.SameKeycards)) - s.Require().True(accounts.Contains(syncedKeycards, keycard2, accounts.SameKeycards)) - s.Require().True(accounts.Contains(syncedKeycards, keycard2Copy, accounts.SameKeycards)) - s.Require().True(accounts.Contains(syncedKeycards, keycard3, accounts.SameKeycards)) -} - -func (s *MessengerBackupSuite) TestBackupWatchOnlyAccounts() { - // Create bob1 - bob1 := s.m - - woAccounts := accounts.GetWatchOnlyAccountsForTest() - err := bob1.settings.SaveOrUpdateAccounts(woAccounts, false) - s.Require().NoError(err) - dbWoAccounts1, err := bob1.settings.GetActiveWatchOnlyAccounts() - s.Require().NoError(err) - s.Require().Equal(len(woAccounts), len(dbWoAccounts1)) - s.Require().True(haveSameElements(woAccounts, dbWoAccounts1, accounts.SameAccounts)) - - // Create bob2 - accountsFeed := &event.Feed{} - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, []Option{WithAccountsFeed(accountsFeed)}) - s.Require().NoError(err) - s.Require().NotNil(bob2.config.accountsFeed) - ch := make(chan accountsevent.Event, 20) - sub := bob2.config.accountsFeed.Subscribe(ch) - defer TearDownMessenger(&s.Suite, bob2) - - // Backup - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - c, err := bob2.settings.GetActiveWatchOnlyAccounts() - if err != nil { - return false - } - return r.BackupHandled && len(woAccounts) == len(c) - - }, - "no messages", - ) - s.Require().NoError(err) - - dbWoAccounts2, err := bob2.settings.GetActiveWatchOnlyAccounts() - s.Require().NoError(err) - s.Require().Equal(len(woAccounts), len(dbWoAccounts2)) - s.Require().True(haveSameElements(woAccounts, dbWoAccounts2, accounts.SameAccounts)) - - // Check whether accounts added event is sent - select { - case <-time.After(1 * time.Second): - s.Fail("Timed out waiting for accountsevent") - case event := <-ch: - switch event.Type { - case accountsevent.EventTypeAdded: - s.Require().Len(event.Accounts, 1) - s.Require().Equal(common.Address(dbWoAccounts2[0].Address), event.Accounts[0]) - } - } - sub.Unsubscribe() -} - -func (s *MessengerBackupSuite) TestBackupChats() { - // Create bob1 - bob1 := s.m - - response, err := bob1.CreateGroupChatWithMembers(context.Background(), "group", []string{}) - s.NoError(err) - s.Require().Len(response.Chats(), 1) - - ourGroupChat := response.Chats()[0] - - err = bob1.SaveChat(ourGroupChat) - s.NoError(err) - - alice := s.newMessenger() - defer TearDownMessenger(&s.Suite, alice) - - ourOneOneChat := CreateOneToOneChat("Our 1TO1", &alice.identity.PublicKey, alice.getTimesource()) - err = bob1.SaveChat(ourOneOneChat) - s.Require().NoError(err) - - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - // Backup - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - // Wait for the message to reach its destination - response, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - s.Require().NoError(err) - - // -- Group chat - // Check response - chat, ok := response.chats[ourGroupChat.ID] - s.Require().True(ok) - s.Require().Equal(ourGroupChat.Name, chat.Name) - // Check stored chats - chat, ok = bob2.allChats.Load(ourGroupChat.ID) - s.Require().True(ok) - s.Require().Equal(ourGroupChat.Name, chat.Name) - - // -- One to One chat - // Check response - chat, ok = response.chats[ourOneOneChat.ID] - s.Require().True(ok) - s.Require().Equal("", chat.Name) // We set 1-1 chat names to "" because the name is not good - // Check stored chats - chat, ok = bob2.allChats.Load(ourOneOneChat.ID) - s.Require().True(ok) - s.Require().Equal("", chat.Name) -} - -func (s *MessengerBackupSuite) TestLeftCommunitiesAreBackedUp() { - bob1 := s.m - // Create bob2 - bob2, err := newMessengerWithKey(s.shh, bob1.identity, s.logger, nil) - s.Require().NoError(err) - defer TearDownMessenger(&s.Suite, bob2) - - description := &requests.CreateCommunity{ - Membership: protobuf.CommunityPermissions_MANUAL_ACCEPT, - Name: "other-status", - Color: "#fffff4", - Description: "other status community description", - } - - // Create another community chat - response, err := bob1.CreateCommunity(description, true) - s.Require().NoError(err) - s.Require().NotNil(response) - s.Require().Len(response.Communities(), 1) - - newCommunity := response.Communities()[0] - - response, err = bob1.LeaveCommunity(newCommunity.ID()) - s.Require().NoError(err) - s.Require().NotNil(response) - - // trigger artificial Backup - _, err = bob1.BackupData(context.Background()) - s.Require().NoError(err) - - communities, err := bob1.Communities() - s.Require().NoError(err) - s.Require().Len(communities, 1) - - // Safety check - communities, err = bob2.Communities() - s.Require().NoError(err) - s.Require().Len(communities, 0) - - // Wait for the message to reach its destination - _, err = WaitOnMessengerResponse( - bob2, - func(r *MessengerResponse) bool { - return r.BackupHandled - }, - "no messages", - ) - - s.Require().NoError(err) - - communities, err = bob2.JoinedCommunities() - s.Require().NoError(err) - s.Require().Len(communities, 0) -} diff --git a/protocol/messenger_browsers_test.go b/protocol/messenger_browsers_test.go index 3fb00b5bf81..01b3a05215a 100644 --- a/protocol/messenger_browsers_test.go +++ b/protocol/messenger_browsers_test.go @@ -20,11 +20,9 @@ type BrowserSuite struct { func (s *BrowserSuite) SetupTest() { s.MessengerBaseTestSuite.SetupTest() - _, err := s.m.Start() - s.Require().NoError(err) } -func (s *MessengerBackupSuite) TestBrowsersOrderedNewestFirst() { +func (s *BrowserSuite) TestBrowsersOrderedNewestFirst() { testBrowsers := []*browsers.Browser{ { ID: "1", @@ -60,7 +58,7 @@ func (s *MessengerBackupSuite) TestBrowsersOrderedNewestFirst() { s.Require().Equal(testBrowsers, rst) } -func (s *MessengerBackupSuite) TestBrowsersHistoryIncluded() { +func (s *BrowserSuite) TestBrowsersHistoryIncluded() { browser := &browsers.Browser{ ID: "1", Name: "first", @@ -76,7 +74,7 @@ func (s *MessengerBackupSuite) TestBrowsersHistoryIncluded() { s.Require().Equal(browser, rst[0]) } -func (s *MessengerBackupSuite) TestBrowsersReplaceOnUpdate() { +func (s *BrowserSuite) TestBrowsersReplaceOnUpdate() { browser := &browsers.Browser{ ID: "1", Name: "first", @@ -95,7 +93,7 @@ func (s *MessengerBackupSuite) TestBrowsersReplaceOnUpdate() { s.Require().Equal(browser, rst[0]) } -func (s *MessengerBackupSuite) TestDeleteBrowser() { +func (s *BrowserSuite) TestDeleteBrowser() { browser := &browsers.Browser{ ID: "1", Name: "first", diff --git a/protocol/messenger_builder_test.go b/protocol/messenger_builder_test.go index 998f120f147..a23974b7186 100644 --- a/protocol/messenger_builder_test.go +++ b/protocol/messenger_builder_test.go @@ -2,23 +2,30 @@ package protocol import ( "crypto/ecdsa" + "encoding/json" + "time" "github.com/google/uuid" + "github.com/pkg/errors" "go.uber.org/zap" - "github.com/status-im/status-go/account/generator" + "github.com/status-im/status-go/accounts-management/generator" "github.com/status-im/status-go/appdatabase" "github.com/status-im/status-go/common/dbsetup" - "github.com/status-im/status-go/eth-node/crypto" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/messaging" + messagingtypes "github.com/status-im/status-go/messaging/types" "github.com/status-im/status-go/multiaccounts" + "github.com/status-im/status-go/multiaccounts/accounts" "github.com/status-im/status-go/multiaccounts/settings" + "github.com/status-im/status-go/params" + "github.com/status-im/status-go/protocol/ens" "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/protocol/sqlite" "github.com/status-im/status-go/protocol/tt" - v1protocol "github.com/status-im/status-go/protocol/v1" + "github.com/status-im/status-go/services/browsers" "github.com/status-im/status-go/t/helpers" "github.com/status-im/status-go/walletdatabase" - - wakutypes "github.com/status-im/status-go/waku/types" ) type testMessengerConfig struct { @@ -29,6 +36,8 @@ type testMessengerConfig struct { unhandledMessagesTracker *unhandledMessagesTracker messagesOrderController *MessagesOrderController + appSettings *settings.Settings + nodeConfig *params.NodeConfig extraOptions []Option } @@ -50,17 +59,29 @@ func (tmc *testMessengerConfig) complete() error { tmc.logger = logger.Named(tmc.name) } + if tmc.appSettings == nil { + tmc.appSettings = newTestSettings() + } + + if tmc.nodeConfig == nil { + tmc.nodeConfig = ¶ms.NodeConfig{} + } + return nil } -func newTestMessenger(waku wakutypes.Waku, config testMessengerConfig) (*Messenger, error) { +func newTestMessenger(messagingEnv *messaging.TestMessagingEnvironment, config testMessengerConfig) (*Messenger, error) { err := config.complete() if err != nil { return nil, err } acc := generator.NewAccount(config.privateKey, nil) - iai := acc.ToIdentifiedAccountInfo("") + iai := acc.ToIdentifiedAccountInfo() + multiAcc := &multiaccounts.Account{ + Timestamp: time.Now().Unix(), + KeyUID: iai.KeyUID, + } madb, err := multiaccounts.InitializeDB(dbsetup.InMemoryPath) if err != nil { @@ -75,26 +96,73 @@ func newTestMessenger(waku wakutypes.Waku, config testMessengerConfig) (*Messeng return nil, err } + err = sqlite.Migrate(appDb) + if err != nil { + return nil, errors.Wrap(err, "failed to apply migrations") + } + + if config.appSettings.Networks == nil { + networks := new(json.RawMessage) + if err := networks.UnmarshalJSON([]byte("net")); err != nil { + return nil, err + } + + config.appSettings.Networks = networks + } + + sDB, err := accounts.NewDB(appDb) + if err != nil { + return nil, err + } + + err = sDB.CreateSettings(*config.appSettings, *config.nodeConfig) + if err != nil { + return nil, err + } + + installationID := uuid.New().String() + + messaging, err := messagingEnv.NewTestCore( + messaging.CoreParams{ + Identity: config.privateKey, + DB: appDb, + Persistence: NewMessagingPersistence(appDb), + InstallationID: installationID, + TimeSource: &testTimeSource{}, + }, + messaging.WithLogger(config.logger), + ) + if err != nil { + return nil, err + } + + ensVerifier := ens.New( + config.logger, + &testTimeSource{}, + appDb, + "", + "", + ) + options := []Option{ WithCustomLogger(config.logger), WithDatabase(appDb), WithWalletDatabase(walletDb), + WithBrowserDatabase(browsers.NewDB(appDb)), WithMultiAccounts(madb), - WithAccount(iai.ToMultiAccount()), + WithAccount(multiAcc), WithDatasync(), - WithToplevelDatabaseMigrations(), - WithBrowserDatabase(nil), WithCuratedCommunitiesUpdateLoop(false), WithStubOnlineChecker(), + WithENSVerifier(ensVerifier), + WithMessageSigner(NewSignerStub()), } options = append(options, config.extraOptions...) m, err := NewMessenger( - config.name, config.privateKey, - waku, - uuid.New().String(), - "testVersion", + messaging.API(), + installationID, options..., ) if err != nil { @@ -119,7 +187,21 @@ func newTestMessenger(waku wakutypes.Waku, config testMessengerConfig) (*Messeng return nil, err } - err = m.InitFilters() + return m, nil +} + +func newRunningTestMessenger(messagingEnv *messaging.TestMessagingEnvironment, config testMessengerConfig) (*Messenger, error) { + m, err := newTestMessenger(messagingEnv, config) + if err != nil { + return nil, err + } + + err = m.messaging.Start() + if err != nil { + return nil, err + } + + _, err = m.Start() if err != nil { return nil, err } @@ -128,7 +210,7 @@ func newTestMessenger(waku wakutypes.Waku, config testMessengerConfig) (*Messeng } type unhandedMessage struct { - *v1protocol.StatusMessage + *messagingtypes.Message err error } @@ -136,7 +218,7 @@ type unhandledMessagesTracker struct { messages map[protobuf.ApplicationMetadataMessage_Type][]*unhandedMessage } -func (u *unhandledMessagesTracker) addMessage(msg *v1protocol.StatusMessage, err error) { +func (u *unhandledMessagesTracker) addMessage(msg *messagingtypes.Message, err error) { msgType := msg.ApplicationLayer.Type if _, exists := u.messages[msgType]; !exists { @@ -144,14 +226,14 @@ func (u *unhandledMessagesTracker) addMessage(msg *v1protocol.StatusMessage, err } newMessage := &unhandedMessage{ - StatusMessage: msg, - err: err, + Message: msg, + err: err, } u.messages[msgType] = append(u.messages[msgType], newMessage) } -func newTestSettings() settings.Settings { - return settings.Settings{ +func newTestSettings() *settings.Settings { + return &settings.Settings{ DisplayName: DefaultProfileDisplayName, ProfilePicturesShowTo: 1, ProfilePicturesVisibility: 1, diff --git a/protocol/messenger_config.go b/protocol/messenger_config.go index cde90194949..d9d281af2d0 100644 --- a/protocol/messenger_config.go +++ b/protocol/messenger_config.go @@ -2,32 +2,24 @@ package protocol import ( "database/sql" - "encoding/json" "time" - "github.com/ethereum/go-ethereum/event" - - "github.com/status-im/status-go/account" messagingtypes "github.com/status-im/status-go/messaging/types" + "github.com/status-im/status-go/pkg/pubsub" "github.com/status-im/status-go/rpc" "github.com/status-im/status-go/server" "github.com/status-im/status-go/services/browsers" - "github.com/status-im/status-go/wakuv2" "go.uber.org/zap" - "github.com/status-im/status-go/appdatabase/migrations" "github.com/status-im/status-go/multiaccounts" - "github.com/status-im/status-go/multiaccounts/accounts" - "github.com/status-im/status-go/multiaccounts/settings" "github.com/status-im/status-go/params" - "github.com/status-im/status-go/protocol/anonmetrics" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" "github.com/status-im/status-go/protocol/discord" + "github.com/status-im/status-go/protocol/ens" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/pushnotificationclient" - "github.com/status-im/status-go/protocol/pushnotificationserver" "github.com/status-im/status-go/protocol/wakusync" "github.com/status-im/status-go/services/mailservers" "github.com/status-im/status-go/services/wallet" @@ -42,7 +34,6 @@ type MessengerSignalsHandler interface { HistoryRequestStarted(numBatches int) HistoryRequestCompleted() - BackupPerformed(uint64) HistoryArchivesProtocolEnabled() HistoryArchivesProtocolDisabled() CreatingHistoryArchives(communityID string) @@ -88,24 +79,20 @@ type config struct { clusterConfig params.ClusterConfig browserDatabase *browsers.Database torrentConfig *params.TorrentConfig - walletConfig *params.WalletConfig walletService *wallet.Service communityTokensService communities.CommunityTokensServiceInterface httpServer *server.MediaServer rpcClient *rpc.Client tokenManager communities.TokenManager collectiblesManager communities.CollectiblesManager - accountsManager account.Manager - - verifyTransactionClient EthClient - verifyENSURL string - verifyENSContractAddress string + accountsManager AccountsManager + signer communities.MessageSigner - anonMetricsClientConfig *anonmetrics.ClientConfig - anonMetricsServerConfig *anonmetrics.ServerConfig + verifyTransactionClient EthClient + ensVerifier *ens.Verifier - pushNotificationServerConfig *pushnotificationserver.Config pushNotificationClientConfig *pushnotificationclient.Config + pushNotificationServer PushNotificationServer logger *zap.Logger @@ -113,16 +100,12 @@ type config struct { messengerSignalsHandler MessengerSignalsHandler - telemetryServerURL string - telemetrySendPeriod time.Duration - wakuService *wakuv2.Waku - messageResendMinDelay time.Duration messageResendMaxCount int communityManagerOptions []communities.ManagerOption - accountsFeed *event.Feed + accountsPublisher *pubsub.Publisher onlineChecker func() bool } @@ -185,37 +168,6 @@ func WithWalletDatabase(db *sql.DB) Option { } } -func WithToplevelDatabaseMigrations() Option { - return func(c *config) error { - c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error { - return migrations.Migrate(c.appDb, nil) - }) - return nil - } -} - -func WithAppSettings(s settings.Settings, nc params.NodeConfig) Option { - return func(c *config) error { - c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error { - if s.Networks == nil { - networks := new(json.RawMessage) - if err := networks.UnmarshalJSON([]byte("net")); err != nil { - return err - } - - s.Networks = networks - } - - sDB, err := accounts.NewDB(c.appDb) - if err != nil { - return err - } - return sDB.CreateSettings(s, nc) - }) - return nil - } -} - func WithMultiAccounts(ma *multiaccounts.Database) Option { return func(c *config) error { c.multiAccount = ma @@ -240,41 +192,6 @@ func WithAccount(acc *multiaccounts.Account) Option { func WithBrowserDatabase(bd *browsers.Database) Option { return func(c *config) error { c.browserDatabase = bd - if c.browserDatabase == nil { - c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error { - c.browserDatabase = browsers.NewDB(c.appDb) - return nil - }) - } - return nil - } -} - -func WithAnonMetricsClientConfig(anonMetricsClientConfig *anonmetrics.ClientConfig) Option { - return func(c *config) error { - c.anonMetricsClientConfig = anonMetricsClientConfig - return nil - } -} - -func WithAnonMetricsServerConfig(anonMetricsServerConfig *anonmetrics.ServerConfig) Option { - return func(c *config) error { - c.anonMetricsServerConfig = anonMetricsServerConfig - return nil - } -} - -func WithTelemetry(serverURL string, sendPeriod time.Duration) Option { - return func(c *config) error { - c.telemetryServerURL = serverURL - c.telemetrySendPeriod = sendPeriod - return nil - } -} - -func WithPushNotificationServerConfig(pushNotificationServerConfig *pushnotificationserver.Config) Option { - return func(c *config) error { - c.pushNotificationServerConfig = pushNotificationServerConfig return nil } } @@ -300,9 +217,9 @@ func WithPushNotifications() func(c *config) error { } } -func WithCheckingForBackupDisabled() func(c *config) error { +func WithPushNotificationServer(server PushNotificationServer) func(c *config) error { return func(c *config) error { - c.featureFlags.DisableCheckingForBackup = true + c.pushNotificationServer = server return nil } } @@ -328,14 +245,6 @@ func WithSignalsHandler(h MessengerSignalsHandler) Option { } } -func WithENSVerificationConfig(url, address string) Option { - return func(c *config) error { - c.verifyENSURL = url - c.verifyENSContractAddress = address - return nil - } -} - func WithClusterConfig(cc params.ClusterConfig) Option { return func(c *config) error { c.clusterConfig = cc @@ -364,13 +273,6 @@ func WithRPCClient(r *rpc.Client) Option { } } -func WithWalletConfig(wc *params.WalletConfig) Option { - return func(c *config) error { - c.walletConfig = wc - return nil - } -} - func WithMessageCSV(enabled bool) Option { return func(c *config) error { c.outputMessagesCSV = enabled @@ -392,37 +294,37 @@ func WithCommunityTokensService(s communities.CommunityTokensServiceInterface) O } } -func WithWakuService(s *wakuv2.Waku) Option { +func WithTokenManager(tokenManager communities.TokenManager) Option { return func(c *config) error { - c.wakuService = s + c.tokenManager = tokenManager return nil } } -func WithTokenManager(tokenManager communities.TokenManager) Option { +func WithCollectiblesManager(collectiblesManager communities.CollectiblesManager) Option { return func(c *config) error { - c.tokenManager = tokenManager + c.collectiblesManager = collectiblesManager return nil } } -func WithCollectiblesManager(collectiblesManager communities.CollectiblesManager) Option { +func WithAccountsManager(accountsManager AccountsManager) Option { return func(c *config) error { - c.collectiblesManager = collectiblesManager + c.accountsManager = accountsManager return nil } } -func WithAccountManager(accountManager account.Manager) Option { +func WithMessageSigner(signer communities.MessageSigner) Option { return func(c *config) error { - c.accountsManager = accountManager + c.signer = signer return nil } } -func WithAccountsFeed(feed *event.Feed) Option { +func WithAccountsPublisher(publisher *pubsub.Publisher) Option { return func(c *config) error { - c.accountsFeed = feed + c.accountsPublisher = publisher return nil } } @@ -433,3 +335,10 @@ func WithNewsFeed() func(c *config) error { return nil } } + +func WithENSVerifier(ensVerifier *ens.Verifier) func(c *config) error { + return func(c *config) error { + c.ensVerifier = ensVerifier + return nil + } +} diff --git a/protocol/messenger_handler.go b/protocol/messenger_handler.go index a2229ea98d9..adf49ff3cee 100644 --- a/protocol/messenger_handler.go +++ b/protocol/messenger_handler.go @@ -12,6 +12,7 @@ import ( gethcommon "github.com/ethereum/go-ethereum/common" gocommon "github.com/status-im/status-go/common" + "github.com/status-im/status-go/pkg/pubsub" "github.com/status-im/status-go/services/accounts/accountsevent" "github.com/status-im/status-go/services/browsers" "github.com/status-im/status-go/signal" @@ -21,9 +22,9 @@ import ( "github.com/google/uuid" - utils "github.com/status-im/status-go/common" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" + accsmanagementtypes "github.com/status-im/status-go/accounts-management/types" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/images" "github.com/status-im/status-go/multiaccounts/accounts" multiaccountscommon "github.com/status-im/status-go/multiaccounts/common" @@ -31,7 +32,6 @@ import ( walletsettings "github.com/status-im/status-go/multiaccounts/settings_wallet" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" - "github.com/status-im/status-go/protocol/encryption/multidevice" "github.com/status-im/status-go/protocol/peersyncing" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/requests" @@ -66,7 +66,7 @@ var ( // HandleMembershipUpdate updates a Chat instance according to the membership updates. // It retrieves chat, if exists, and merges membership updates from the message. // Finally, the Chat is updated with the new group events. -func (m *Messenger) HandleMembershipUpdateMessage(messageState *ReceivedMessageState, rawMembershipUpdate *protobuf.MembershipUpdateMessage, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleMembershipUpdateMessage(messageState *ReceivedMessageState, rawMembershipUpdate *protobuf.MembershipUpdateMessage, statusMessage *messagingtypes.Message) error { chat, _ := messageState.AllChats.Load(rawMembershipUpdate.ChatId) return m.HandleMembershipUpdate(messageState, chat, rawMembershipUpdate, m.systemMessagesTranslations) @@ -393,7 +393,6 @@ func (m *Messenger) handleCommandMessage(state *ReceivedMessageState, message *c message.From = state.CurrentMessageState.Contact.ID message.Alias = state.CurrentMessageState.Contact.Alias message.SigPubKey = state.CurrentMessageState.PublicKey - message.Identicon = state.CurrentMessageState.Contact.Identicon message.WhisperTimestamp = state.CurrentMessageState.WhisperTimestamp if err := message.PrepareContent(common.PubkeyToHex(&m.identity.PublicKey)); err != nil { @@ -528,7 +527,7 @@ func (m *Messenger) syncContactRequestForInstallationContact(contact *Contact, s return nil } -func (m *Messenger) HandleSyncInstallationAccount(state *ReceivedMessageState, message *protobuf.SyncInstallationAccount, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncInstallationAccount(state *ReceivedMessageState, message *protobuf.SyncInstallationAccount, statusMessage *messagingtypes.Message) error { // Noop return nil } @@ -586,7 +585,7 @@ func (m *Messenger) handleSyncChats(messageState *ReceivedMessageState, chats [] return nil } -func (m *Messenger) HandleSyncInstallationContactV2(state *ReceivedMessageState, message *protobuf.SyncInstallationContactV2, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncInstallationContactV2(state *ReceivedMessageState, message *protobuf.SyncInstallationContactV2, statusMessage *messagingtypes.Message) error { // Ignore own contact installation if message.Id == m.myHexIdentity() { @@ -769,7 +768,7 @@ func (m *Messenger) HandleSyncInstallationContactV2(state *ReceivedMessageState, return nil } -func (m *Messenger) HandleSyncProfilePictures(state *ReceivedMessageState, message *protobuf.SyncProfilePictures, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncProfilePictures(state *ReceivedMessageState, message *protobuf.SyncProfilePictures, statusMessage *messagingtypes.Message) error { dbImages, err := m.multiAccounts.GetIdentityImages(message.KeyUid) if err != nil { return err @@ -809,7 +808,7 @@ func (m *Messenger) HandleSyncProfilePictures(state *ReceivedMessageState, messa return err } -func (m *Messenger) HandleSyncChat(state *ReceivedMessageState, message *protobuf.SyncChat, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncChat(state *ReceivedMessageState, message *protobuf.SyncChat, statusMessage *messagingtypes.Message) error { chatID := message.Id existingChat, ok := state.AllChats.Load(chatID) if ok && (existingChat.Active || uint32(message.GetClock()/1000) < existingChat.SyncedTo) { @@ -830,7 +829,7 @@ func (m *Messenger) HandleSyncChat(state *ReceivedMessageState, message *protobu return nil } -func (m *Messenger) HandleSyncChatRemoved(state *ReceivedMessageState, message *protobuf.SyncChatRemoved, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncChatRemoved(state *ReceivedMessageState, message *protobuf.SyncChatRemoved, statusMessage *messagingtypes.Message) error { chat, ok := m.allChats.Load(message.Id) if !ok { return ErrChatNotFound @@ -859,7 +858,7 @@ func (m *Messenger) HandleSyncChatRemoved(state *ReceivedMessageState, message * return state.Response.Merge(response) } -func (m *Messenger) HandleSyncChatMessagesRead(state *ReceivedMessageState, message *protobuf.SyncChatMessagesRead, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncChatMessagesRead(state *ReceivedMessageState, message *protobuf.SyncChatMessagesRead, statusMessage *messagingtypes.Message) error { chat, ok := m.allChats.Load(message.Id) if !ok { return ErrChatNotFound @@ -894,7 +893,6 @@ func (m *Messenger) handlePinMessage(pinner *Contact, whisperTimestamp uint64, r WhisperTimestamp: whisperTimestamp, From: pinner.ID, SigPubKey: publicKey, - Identicon: pinner.Identicon, Alias: pinner.Alias, } @@ -971,7 +969,7 @@ func (m *Messenger) handlePinMessage(pinner *Contact, whisperTimestamp uint64, r return nil } -func (m *Messenger) HandlePinMessage(state *ReceivedMessageState, message *protobuf.PinMessage, statusMessage *v1protocol.StatusMessage, fromArchive bool) error { +func (m *Messenger) HandlePinMessage(state *ReceivedMessageState, message *protobuf.PinMessage, statusMessage *messagingtypes.Message, fromArchive bool) error { return m.handlePinMessage(state.CurrentMessageState.Contact, state.CurrentMessageState.WhisperTimestamp, state.Response, message, fromArchive) } @@ -1118,7 +1116,7 @@ func (m *Messenger) handleAcceptContactRequestMessage(state *ReceivedMessageStat return nil } -func (m *Messenger) HandleAcceptContactRequest(state *ReceivedMessageState, message *protobuf.AcceptContactRequest, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleAcceptContactRequest(state *ReceivedMessageState, message *protobuf.AcceptContactRequest, statusMessage *messagingtypes.Message) error { err := m.handleAcceptContactRequestMessage(state, message.Clock, message.Id, false) if err != nil { m.logger.Warn("could not accept contact request", zap.Error(err)) @@ -1197,7 +1195,7 @@ func (m *Messenger) handleRetractContactRequest(state *ReceivedMessageState, con return nil } -func (m *Messenger) HandleRetractContactRequest(state *ReceivedMessageState, message *protobuf.RetractContactRequest, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleRetractContactRequest(state *ReceivedMessageState, message *protobuf.RetractContactRequest, statusMessage *messagingtypes.Message) error { contact := state.CurrentMessageState.Contact err := m.handleRetractContactRequest(state, contact, message) if err != nil { @@ -1210,7 +1208,7 @@ func (m *Messenger) HandleRetractContactRequest(state *ReceivedMessageState, mes return nil } -func (m *Messenger) HandleContactUpdate(state *ReceivedMessageState, message *protobuf.ContactUpdate, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleContactUpdate(state *ReceivedMessageState, message *protobuf.ContactUpdate, statusMessage *messagingtypes.Message) error { logger := m.logger.With(zap.String("site", "HandleContactUpdate")) if common.IsPubKeyEqual(state.CurrentMessageState.PublicKey, &m.identity.PublicKey) { @@ -1229,7 +1227,7 @@ func (m *Messenger) HandleContactUpdate(state *ReceivedMessageState, message *pr return ErrMessageNotAllowed } - if err = utils.ValidateDisplayName(&message.DisplayName); err != nil { + if err = gocommon.ValidateDisplayName(&message.DisplayName); err != nil { return err } @@ -1310,7 +1308,7 @@ func (m *Messenger) HandleContactUpdate(state *ReceivedMessageState, message *pr return nil } -func (m *Messenger) HandleSyncPairInstallation(state *ReceivedMessageState, message *protobuf.SyncPairInstallation, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncPairInstallation(state *ReceivedMessageState, message *protobuf.SyncPairInstallation, statusMessage *messagingtypes.Message) error { logger := m.logger.With(zap.String("site", "HandlePairInstallation")) if err := ValidateReceivedPairInstallation(message, state.CurrentMessageState.WhisperTimestamp); err != nil { logger.Warn("failed to validate message", zap.Error(err)) @@ -1322,7 +1320,7 @@ func (m *Messenger) HandleSyncPairInstallation(state *ReceivedMessageState, mess return errors.New("installation not found") } - metadata := &multidevice.InstallationMetadata{ + metadata := &messagingtypes.InstallationMetadata{ Name: message.Name, DeviceType: message.DeviceType, } @@ -1504,7 +1502,7 @@ func (m *Messenger) handleArchiveMessages(archiveMessages []*protobuf.WakuMessag return response, nil } -func (m *Messenger) HandleCommunityCancelRequestToJoin(state *ReceivedMessageState, cancelRequestToJoinProto *protobuf.CommunityCancelRequestToJoin, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleCommunityCancelRequestToJoin(state *ReceivedMessageState, cancelRequestToJoinProto *protobuf.CommunityCancelRequestToJoin, statusMessage *messagingtypes.Message) error { signer := state.CurrentMessageState.PublicKey if cancelRequestToJoinProto.CommunityId == nil { return ErrInvalidCommunityID @@ -1545,7 +1543,7 @@ func (m *Messenger) HandleCommunityCancelRequestToJoin(state *ReceivedMessageSta } // HandleCommunityRequestToJoin handles an community request to join -func (m *Messenger) HandleCommunityRequestToJoin(state *ReceivedMessageState, requestToJoinProto *protobuf.CommunityRequestToJoin, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleCommunityRequestToJoin(state *ReceivedMessageState, requestToJoinProto *protobuf.CommunityRequestToJoin, statusMessage *messagingtypes.Message) error { signer := state.CurrentMessageState.PublicKey community, requestToJoin, err := m.communitiesManager.HandleCommunityRequestToJoin(signer, statusMessage.TransportLayer.Dst, requestToJoinProto) if err != nil { @@ -1634,7 +1632,7 @@ func (m *Messenger) HandleCommunityRequestToJoin(state *ReceivedMessageState, re } // HandleCommunityEditSharedAddresses handles an edit a user has made to their shared addresses -func (m *Messenger) HandleCommunityEditSharedAddresses(state *ReceivedMessageState, editRevealedAddressesProto *protobuf.CommunityEditSharedAddresses, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleCommunityEditSharedAddresses(state *ReceivedMessageState, editRevealedAddressesProto *protobuf.CommunityEditSharedAddresses, statusMessage *messagingtypes.Message) error { signer := state.CurrentMessageState.PublicKey if editRevealedAddressesProto.CommunityId == nil { return ErrInvalidCommunityID @@ -1654,7 +1652,7 @@ func (m *Messenger) HandleCommunityEditSharedAddresses(state *ReceivedMessageSta return nil } -func (m *Messenger) HandleCommunityRequestToJoinResponse(state *ReceivedMessageState, requestToJoinResponseProto *protobuf.CommunityRequestToJoinResponse, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleCommunityRequestToJoinResponse(state *ReceivedMessageState, requestToJoinResponseProto *protobuf.CommunityRequestToJoinResponse, statusMessage *messagingtypes.Message) error { signer := state.CurrentMessageState.PublicKey if requestToJoinResponseProto.CommunityId == nil { return ErrInvalidCommunityID @@ -1783,7 +1781,7 @@ func (m *Messenger) HandleCommunityRequestToJoinResponse(state *ReceivedMessageS return nil } -func (m *Messenger) HandleCommunityRequestToLeave(state *ReceivedMessageState, requestToLeaveProto *protobuf.CommunityRequestToLeave, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleCommunityRequestToLeave(state *ReceivedMessageState, requestToLeaveProto *protobuf.CommunityRequestToLeave, statusMessage *messagingtypes.Message) error { signer := state.CurrentMessageState.PublicKey if requestToLeaveProto.CommunityId == nil { return ErrInvalidCommunityID @@ -1897,7 +1895,7 @@ func (m *Messenger) handleEditMessage(state *ReceivedMessageState, editMessage E return nil } -func (m *Messenger) HandleEditMessage(state *ReceivedMessageState, editProto *protobuf.EditMessage, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleEditMessage(state *ReceivedMessageState, editProto *protobuf.EditMessage, statusMessage *messagingtypes.Message) error { return m.handleEditMessage(state, EditMessage{ EditMessage: editProto, From: state.CurrentMessageState.Contact.ID, @@ -2038,7 +2036,7 @@ func (m *Messenger) handleDeleteMessage(state *ReceivedMessageState, deleteMessa return nil } -func (m *Messenger) HandleDeleteMessage(state *ReceivedMessageState, deleteProto *protobuf.DeleteMessage, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleDeleteMessage(state *ReceivedMessageState, deleteProto *protobuf.DeleteMessage, statusMessage *messagingtypes.Message) error { return m.handleDeleteMessage(state, &DeleteMessage{ DeleteMessage: deleteProto, From: state.CurrentMessageState.Contact.ID, @@ -2057,7 +2055,7 @@ func (m *Messenger) getMessageFromResponseOrDatabase(response *MessengerResponse return m.persistence.MessageByID(messageID) } -func (m *Messenger) HandleSyncDeleteForMeMessage(state *ReceivedMessageState, deleteForMeMessage *protobuf.SyncDeleteForMeMessage, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncDeleteForMeMessage(state *ReceivedMessageState, deleteForMeMessage *protobuf.SyncDeleteForMeMessage, statusMessage *messagingtypes.Message) error { if err := ValidateDeleteForMeMessage(deleteForMeMessage); err != nil { return err } @@ -2150,7 +2148,6 @@ func (m *Messenger) handleChatMessage(state *ReceivedMessageState, forceSeen boo From: state.CurrentMessageState.Contact.ID, Alias: state.CurrentMessageState.Contact.Alias, SigPubKey: state.CurrentMessageState.PublicKey, - Identicon: state.CurrentMessageState.Contact.Identicon, WhisperTimestamp: state.CurrentMessageState.WhisperTimestamp, } @@ -2487,7 +2484,7 @@ func (m *Messenger) handleChatMessage(state *ReceivedMessageState, forceSeen boo return nil } -func (m *Messenger) addPeersyncingMessage(chat *Chat, msg *v1protocol.StatusMessage) error { +func (m *Messenger) addPeersyncingMessage(chat *Chat, msg *messagingtypes.Message) error { if msg == nil { return nil } @@ -2509,12 +2506,12 @@ func (m *Messenger) addPeersyncingMessage(chat *Chat, msg *v1protocol.StatusMess return m.peersyncing.Add(syncMessage) } -func (m *Messenger) HandleChatMessage(state *ReceivedMessageState, message *protobuf.ChatMessage, statusMessage *v1protocol.StatusMessage, fromArchive bool) error { +func (m *Messenger) HandleChatMessage(state *ReceivedMessageState, message *protobuf.ChatMessage, statusMessage *messagingtypes.Message, fromArchive bool) error { state.CurrentMessageState.Message = message return m.handleChatMessage(state, fromArchive) } -func (m *Messenger) HandleRequestAddressForTransaction(messageState *ReceivedMessageState, command *protobuf.RequestAddressForTransaction, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleRequestAddressForTransaction(messageState *ReceivedMessageState, command *protobuf.RequestAddressForTransaction, statusMessage *messagingtypes.Message) error { err := ValidateReceivedRequestAddressForTransaction(command, messageState.CurrentMessageState.WhisperTimestamp) if err != nil { return err @@ -2539,7 +2536,7 @@ func (m *Messenger) HandleRequestAddressForTransaction(messageState *ReceivedMes return m.handleCommandMessage(messageState, message) } -func (m *Messenger) HandleSyncSetting(messageState *ReceivedMessageState, message *protobuf.SyncSetting, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncSetting(messageState *ReceivedMessageState, message *protobuf.SyncSetting, statusMessage *messagingtypes.Message) error { settingField, err := m.extractAndSaveSyncSetting(message) if err != nil { return err @@ -2570,7 +2567,7 @@ func (m *Messenger) HandleSyncSetting(messageState *ReceivedMessageState, messag return nil } -func (m *Messenger) HandleSyncAccountCustomizationColor(state *ReceivedMessageState, message *protobuf.SyncAccountCustomizationColor, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncAccountCustomizationColor(state *ReceivedMessageState, message *protobuf.SyncAccountCustomizationColor, statusMessage *messagingtypes.Message) error { affected, err := m.multiAccounts.UpdateAccountCustomizationColor(message.GetKeyUid(), message.GetCustomizationColor(), message.GetUpdatedAt()) if err != nil { return err @@ -2583,7 +2580,7 @@ func (m *Messenger) HandleSyncAccountCustomizationColor(state *ReceivedMessageSt return nil } -func (m *Messenger) HandleRequestTransaction(messageState *ReceivedMessageState, command *protobuf.RequestTransaction, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleRequestTransaction(messageState *ReceivedMessageState, command *protobuf.RequestTransaction, statusMessage *messagingtypes.Message) error { err := ValidateReceivedRequestTransaction(command, messageState.CurrentMessageState.WhisperTimestamp) if err != nil { return err @@ -2609,7 +2606,7 @@ func (m *Messenger) HandleRequestTransaction(messageState *ReceivedMessageState, return m.handleCommandMessage(messageState, message) } -func (m *Messenger) HandleAcceptRequestAddressForTransaction(messageState *ReceivedMessageState, command *protobuf.AcceptRequestAddressForTransaction, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleAcceptRequestAddressForTransaction(messageState *ReceivedMessageState, command *protobuf.AcceptRequestAddressForTransaction, statusMessage *messagingtypes.Message) error { err := ValidateReceivedAcceptRequestAddressForTransaction(command, messageState.CurrentMessageState.WhisperTimestamp) if err != nil { return err @@ -2660,7 +2657,7 @@ func (m *Messenger) HandleAcceptRequestAddressForTransaction(messageState *Recei return m.handleCommandMessage(messageState, initialMessage) } -func (m *Messenger) HandleSendTransaction(messageState *ReceivedMessageState, command *protobuf.SendTransaction, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSendTransaction(messageState *ReceivedMessageState, command *protobuf.SendTransaction, statusMessage *messagingtypes.Message) error { err := ValidateReceivedSendTransaction(command, messageState.CurrentMessageState.WhisperTimestamp) if err != nil { return err @@ -2680,7 +2677,7 @@ func (m *Messenger) HandleSendTransaction(messageState *ReceivedMessageState, co return m.persistence.SaveTransactionToValidate(transactionToValidate) } -func (m *Messenger) HandleDeclineRequestAddressForTransaction(messageState *ReceivedMessageState, command *protobuf.DeclineRequestAddressForTransaction, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleDeclineRequestAddressForTransaction(messageState *ReceivedMessageState, command *protobuf.DeclineRequestAddressForTransaction, statusMessage *messagingtypes.Message) error { err := ValidateReceivedDeclineRequestAddressForTransaction(command, messageState.CurrentMessageState.WhisperTimestamp) if err != nil { return err @@ -2722,7 +2719,7 @@ func (m *Messenger) HandleDeclineRequestAddressForTransaction(messageState *Rece return m.handleCommandMessage(messageState, oldMessage) } -func (m *Messenger) HandleDeclineRequestTransaction(messageState *ReceivedMessageState, command *protobuf.DeclineRequestTransaction, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleDeclineRequestTransaction(messageState *ReceivedMessageState, command *protobuf.DeclineRequestTransaction, statusMessage *messagingtypes.Message) error { err := ValidateReceivedDeclineRequestTransaction(command, messageState.CurrentMessageState.WhisperTimestamp) if err != nil { return err @@ -2764,7 +2761,7 @@ func (m *Messenger) HandleDeclineRequestTransaction(messageState *ReceivedMessag return m.handleCommandMessage(messageState, oldMessage) } -func (m *Messenger) matchChatEntity(chatEntity common.ChatEntity, messageType protobuf.ApplicationMetadataMessage_Type) (*Chat, error) { +func (m *Messenger) matchChatEntity(chatEntity messagingtypes.ChatEntity, messageType protobuf.ApplicationMetadataMessage_Type) (*Chat, error) { if chatEntity.GetSigPubKey() == nil { m.logger.Error("public key can't be empty") return nil, errors.New("received a chatEntity with empty public key") @@ -2895,7 +2892,7 @@ func (m *Messenger) messageExists(messageID string, existingMessagesMap map[stri return false, nil } -func (m *Messenger) HandleEmojiReaction(state *ReceivedMessageState, pbEmojiR *protobuf.EmojiReaction, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleEmojiReaction(state *ReceivedMessageState, pbEmojiR *protobuf.EmojiReaction, statusMessage *messagingtypes.Message) error { logger := m.logger.With(zap.String("site", "HandleEmojiReaction")) if err := ValidateReceivedEmojiReaction(pbEmojiR, state.Timesource.GetCurrentTime()); err != nil { logger.Error("invalid emoji reaction", zap.Error(err)) @@ -2949,7 +2946,7 @@ func (m *Messenger) HandleEmojiReaction(state *ReceivedMessageState, pbEmojiR *p return nil } -func (m *Messenger) HandleGroupChatInvitation(state *ReceivedMessageState, pbGHInvitations *protobuf.GroupChatInvitation, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleGroupChatInvitation(state *ReceivedMessageState, pbGHInvitations *protobuf.GroupChatInvitation, statusMessage *messagingtypes.Message) error { allowed, err := m.isMessageAllowedFrom(state.CurrentMessageState.Contact.ID, nil) if err != nil { return err @@ -2999,7 +2996,7 @@ func (m *Messenger) HandleGroupChatInvitation(state *ReceivedMessageState, pbGHI return nil } -func (m *Messenger) HandleContactCodeAdvertisement(state *ReceivedMessageState, cca *protobuf.ContactCodeAdvertisement, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleContactCodeAdvertisement(state *ReceivedMessageState, cca *protobuf.ContactCodeAdvertisement, statusMessage *messagingtypes.Message) error { if cca.ChatIdentity == nil { return nil } @@ -3008,7 +3005,7 @@ func (m *Messenger) HandleContactCodeAdvertisement(state *ReceivedMessageState, // HandleChatIdentity handles an incoming protobuf.ChatIdentity // extracts contact information stored in the protobuf and adds it to the user's contact for update. -func (m *Messenger) HandleChatIdentity(state *ReceivedMessageState, ci *protobuf.ChatIdentity, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleChatIdentity(state *ReceivedMessageState, ci *protobuf.ChatIdentity, statusMessage *messagingtypes.Message) error { s, err := m.settings.GetSettings() if err != nil { return err @@ -3084,7 +3081,7 @@ func (m *Messenger) HandleChatIdentity(state *ReceivedMessageState, ci *protobuf } if clockChanged { - if err = utils.ValidateDisplayName(&ci.DisplayName); err != nil { + if err = gocommon.ValidateDisplayName(&ci.DisplayName); err != nil { return err } @@ -3124,7 +3121,7 @@ func (m *Messenger) HandleChatIdentity(state *ReceivedMessageState, ci *protobuf return nil } -func (m *Messenger) HandleAnonymousMetricBatch(state *ReceivedMessageState, amb *protobuf.AnonymousMetricBatch, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleAnonymousMetricBatch(state *ReceivedMessageState, amb *protobuf.AnonymousMetricBatch, statusMessage *messagingtypes.Message) error { // TODO return nil @@ -3288,8 +3285,8 @@ func (m *Messenger) updateUnviewedCounts(chat *Chat, message *common.Message) { } } -func mapSyncAccountToAccount(message *protobuf.SyncAccount, accountOperability accounts.AccountOperable, accType accounts.AccountType) *accounts.Account { - return &accounts.Account{ +func mapSyncAccountToAccount(message *protobuf.SyncAccount, accountOperability accsmanagementtypes.AccountOperable, accType accsmanagementtypes.AccountType) *accsmanagementtypes.Account { + return &accsmanagementtypes.Account{ Address: types.BytesToAddress(message.Address), KeyUID: message.KeyUid, PublicKey: types.HexBytes(message.PublicKey), @@ -3310,17 +3307,17 @@ func mapSyncAccountToAccount(message *protobuf.SyncAccount, accountOperability a } } -func (m *Messenger) resolveAccountOperability(syncAcc *protobuf.SyncAccount, recoverinrecoveringFromWakuInitiatedByKeycard bool, - syncKpMigratedToKeycard bool, dbKpMigratedToKeycard bool, accountReceivedFromLocalPairing bool) (accounts.AccountOperable, error) { +func (m *Messenger) resolveAccountOperability(syncAcc *protobuf.SyncAccount, + syncKpMigratedToKeycard bool, dbKpMigratedToKeycard bool, accountReceivedFromLocalPairing bool) (accsmanagementtypes.AccountOperable, error) { if accountReceivedFromLocalPairing { - return accounts.AccountOperable(syncAcc.Operable), nil + return accsmanagementtypes.AccountOperable(syncAcc.Operable), nil } - if syncKpMigratedToKeycard || recoverinrecoveringFromWakuInitiatedByKeycard && m.account.KeyUID == syncAcc.KeyUid { - return accounts.AccountFullyOperable, nil + if syncKpMigratedToKeycard && m.account.KeyUID == syncAcc.KeyUid { + return accsmanagementtypes.AccountFullyOperable, nil } - accountsOperability := accounts.AccountNonOperable + accountsOperability := accsmanagementtypes.AccountNonOperable dbAccount, err := m.settings.GetAccountByAddress(types.BytesToAddress(syncAcc.Address)) if err != nil && err != accounts.ErrDbAccountNotFound { return accountsOperability, err @@ -3328,7 +3325,7 @@ func (m *Messenger) resolveAccountOperability(syncAcc *protobuf.SyncAccount, rec if dbAccount != nil { // We're here when we receive a keypair from the paired device which has just migrated from keycard to app. if !syncKpMigratedToKeycard && dbKpMigratedToKeycard { - return accounts.AccountNonOperable, nil + return accsmanagementtypes.AccountNonOperable, nil } return dbAccount.Operable, nil } @@ -3339,39 +3336,39 @@ func (m *Messenger) resolveAccountOperability(syncAcc *protobuf.SyncAccount, rec // 2. was just converted from keycard to a regular keypair. dbKeycardsForKeyUID, err := m.settings.GetKeycardsWithSameKeyUID(syncAcc.KeyUid) if err != nil { - return accounts.AccountNonOperable, err + return accsmanagementtypes.AccountNonOperable, err } if len(dbKeycardsForKeyUID) > 0 { // We're here in case 2. from above and in this case we need to mark all accounts for this keypair non operable - return accounts.AccountNonOperable, nil + return accsmanagementtypes.AccountNonOperable, nil } } if syncAcc.Chat || syncAcc.Wallet { - accountsOperability = accounts.AccountFullyOperable + accountsOperability = accsmanagementtypes.AccountFullyOperable } else { partiallyOrFullyOperable, err := m.settings.IsAnyAccountPartiallyOrFullyOperableForKeyUID(syncAcc.KeyUid) if err != nil { - if err == accounts.ErrDbKeypairNotFound { - return accounts.AccountNonOperable, nil + if err == accsmanagementtypes.ErrDbKeypairNotFound { + return accsmanagementtypes.AccountNonOperable, nil } - return accounts.AccountNonOperable, err + return accsmanagementtypes.AccountNonOperable, err } if partiallyOrFullyOperable { - accountsOperability = accounts.AccountPartiallyOperable + accountsOperability = accsmanagementtypes.AccountPartiallyOperable } } return accountsOperability, nil } -func (m *Messenger) handleSyncWatchOnlyAccount(message *protobuf.SyncAccount, fromBackup bool) (*accounts.Account, error) { +func (m *Messenger) handleSyncWatchOnlyAccount(message *protobuf.SyncAccount) (*accsmanagementtypes.Account, error) { if message.KeyUid != "" { return nil, ErrNotWatchOnlyAccount } - accountOperability := accounts.AccountFullyOperable + accountOperability := accsmanagementtypes.AccountFullyOperable accAddress := types.BytesToAddress(message.Address) dbAccount, err := m.settings.GetAccountByAddress(accAddress) @@ -3389,36 +3386,34 @@ func (m *Messenger) handleSyncWatchOnlyAccount(message *protobuf.SyncAccount, fr if err != nil { return nil, err } - // if keypair is retrieved from backed up data, no need for resolving accounts positions - if !fromBackup { - err = m.settings.ResolveAccountsPositions(message.Clock) - if err != nil { - return nil, err - } + + err = m.settings.ResolveAccountsPositions(message.Clock) + if err != nil { + return nil, err } dbAccount.Removed = true return dbAccount, nil } } - acc := mapSyncAccountToAccount(message, accountOperability, accounts.AccountTypeWatch) + acc := mapSyncAccountToAccount(message, accountOperability, accsmanagementtypes.AccountTypeWatch) - err = m.settings.SaveOrUpdateAccounts([]*accounts.Account{acc}, false) + err = m.settings.SaveOrUpdateAccounts([]*accsmanagementtypes.Account{acc}, false) if err != nil { return nil, err } - if m.config.accountsFeed != nil { - var eventType accountsevent.EventType + if m.config.accountsPublisher != nil { + payload := []gethcommon.Address{gethcommon.Address(acc.Address)} if acc.Removed { - eventType = accountsevent.EventTypeRemoved + pubsub.Publish(m.config.accountsPublisher, accountsevent.AccountsRemovedEvent{ + Accounts: payload, + }) } else { - eventType = accountsevent.EventTypeAdded + pubsub.Publish(m.config.accountsPublisher, accountsevent.AccountsAddedEvent{ + Accounts: payload, + }) } - m.config.accountsFeed.Send(accountsevent.Event{ - Type: eventType, - Accounts: []gethcommon.Address{gethcommon.Address(acc.Address)}, - }) } return acc, nil } @@ -3507,7 +3502,7 @@ func (m *Messenger) handleSyncCollectiblePreferences(message *protobuf.SyncColle return collectiblePreferences, nil } -func (m *Messenger) handleSyncAccountsPositions(message *protobuf.SyncAccountsPositions) ([]*accounts.Account, error) { +func (m *Messenger) handleSyncAccountsPositions(message *protobuf.SyncAccountsPositions) ([]*accsmanagementtypes.Account, error) { if len(message.Accounts) == 0 { return nil, nil } @@ -3523,9 +3518,9 @@ func (m *Messenger) handleSyncAccountsPositions(message *protobuf.SyncAccountsPo return nil, ErrTryingToApplyOldWalletAccountsOrder } - var accs []*accounts.Account + var accs []*accsmanagementtypes.Account for _, sAcc := range message.Accounts { - acc := &accounts.Account{ + acc := &accsmanagementtypes.Account{ Address: types.BytesToAddress(sAcc.Address), KeyUID: sAcc.KeyUid, Position: sAcc.Position, @@ -3575,19 +3570,19 @@ func (m *Messenger) handleProfileKeypairMigration(state *ReceivedMessageState, f return migrationNeeded, nil } -func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPairing bool, acNofificationCallback func() error) (*accounts.Keypair, error) { +func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPairing bool, acNofificationCallback func() error) (*accsmanagementtypes.Keypair, error) { if message == nil { return nil, errors.New("handleSyncKeypair receive a nil message") } dbKeypair, err := m.settings.GetKeypairByKeyUID(message.KeyUid) - if err != nil && err != accounts.ErrDbKeypairNotFound { + if err != nil && err != accsmanagementtypes.ErrDbKeypairNotFound { return nil, err } - kp := &accounts.Keypair{ + kp := &accsmanagementtypes.Keypair{ KeyUID: message.KeyUid, Name: message.Name, - Type: accounts.KeypairType(message.Type), + Type: accsmanagementtypes.KeypairType(message.Type), DerivedFrom: message.DerivedFrom, LastUsedDerivationIndex: message.LastUsedDerivationIndex, SyncedFrom: message.SyncedFrom, @@ -3595,6 +3590,8 @@ func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPa Removed: message.Removed, } + oldAddresses := make(map[types.Address]bool) + if dbKeypair != nil { if dbKeypair.Clock >= kp.Clock { return nil, ErrTryingToStoreOldKeypair @@ -3603,81 +3600,48 @@ func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPa // but in case if keypair on this device came from the backup (e.g. device A recovered from waku, then device B paired with the device A // via local pairing, before device A made its keypairs fully operable) we need to update syncedFrom when user on this device when that // keypair becomes operable on any of other paired devices - if dbKeypair.SyncedFrom != accounts.SyncedFromBackup { - kp.SyncedFrom = dbKeypair.SyncedFrom + kp.SyncedFrom = dbKeypair.SyncedFrom + for _, acc := range dbKeypair.Accounts { + oldAddresses[acc.Address] = !acc.Removed } } syncKpMigratedToKeycard := len(message.Keycards) > 0 - recoveringFromWaku := message.SyncedFrom == accounts.SyncedFromBackup - multiAcc, err := m.multiAccounts.GetAccount(kp.KeyUID) - if err != nil { - return nil, err - } - recoverinrecoveringFromWakuInitiatedByKeycard := recoveringFromWaku && multiAcc != nil && multiAcc.RefersToKeycard() for _, sAcc := range message.Accounts { accountOperability, err := m.resolveAccountOperability(sAcc, - recoverinrecoveringFromWakuInitiatedByKeycard, syncKpMigratedToKeycard, dbKeypair != nil && dbKeypair.MigratedToKeycard(), fromLocalPairing) if err != nil { return nil, err } - acc := mapSyncAccountToAccount(sAcc, accountOperability, accounts.GetAccountTypeForKeypairType(kp.Type)) + acc := mapSyncAccountToAccount(sAcc, accountOperability, accsmanagementtypes.GetAccountTypeForKeypairType(kp.Type)) kp.Accounts = append(kp.Accounts, acc) } - if !fromLocalPairing && !recoverinrecoveringFromWakuInitiatedByKeycard { - if kp.Removed || - dbKeypair != nil && !dbKeypair.MigratedToKeycard() && syncKpMigratedToKeycard { - // delete all keystore files - err = m.deleteKeystoreFilesForKeypair(dbKeypair) - if err != nil { - return nil, err - } - - if syncKpMigratedToKeycard { - err = m.settings.MarkKeypairFullyOperable(dbKeypair.KeyUID, 0, false) - if err != nil { - return nil, err - } - } - } else if dbKeypair != nil { - for _, dbAcc := range dbKeypair.Accounts { - removeAcc := false - for _, acc := range kp.Accounts { - if dbAcc.Address == acc.Address && acc.Removed && !dbAcc.Removed { - removeAcc = true - break - } - } - if removeAcc { - err = m.deleteKeystoreFileForAddress(dbAcc.Address) - if err != nil { - return nil, err - } - } - } + if !fromLocalPairing && + dbKeypair != nil && + !dbKeypair.MigratedToKeycard() && + syncKpMigratedToKeycard { + err = m.settings.MarkKeypairFullyOperable(dbKeypair.KeyUID, 0, false) + if err != nil { + return nil, err } } // deleting keypair will delete related keycards as well err = m.settings.RemoveKeypair(message.KeyUid, message.Clock) - if err != nil && err != accounts.ErrDbKeypairNotFound { + if err != nil && err != accsmanagementtypes.ErrDbKeypairNotFound { return nil, err } // if entire keypair was removed and keypair is already in db, there is no point to continue if kp.Removed && dbKeypair != nil { - // if keypair is retrieved from backed up data, no need for resolving accounts positions - if message.SyncedFrom != accounts.SyncedFromBackup { - err = m.settings.ResolveAccountsPositions(message.Clock) - if err != nil { - return nil, err - } + err = m.settings.ResolveAccountsPositions(message.Clock) + if err != nil { + return nil, err } return kp, nil } @@ -3688,25 +3652,22 @@ func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPa return nil, err } - // if keypair is retrieved from backed up data, no need for resolving accounts positions - if message.SyncedFrom != accounts.SyncedFromBackup { - // then resolve accounts positions, cause some accounts might be removed - err = m.settings.ResolveAccountsPositions(message.Clock) - if err != nil { - return nil, err - } + // then resolve accounts positions, cause some accounts might be removed + err = m.settings.ResolveAccountsPositions(message.Clock) + if err != nil { + return nil, err + } - // if keypair is coming from paired device (means not from backup) and it's not among known, active keypairs, - // we need to add an activity center notification - if !kp.Removed && dbKeypair == nil { - defer func() { - err = acNofificationCallback() - }() - } + // if keypair is coming from paired device (means not from backup) and it's not among known, active keypairs, + // we need to add an activity center notification + if !kp.Removed && dbKeypair == nil { + defer func() { + err = acNofificationCallback() + }() } for _, sKc := range message.Keycards { - kc := accounts.Keycard{} + kc := accsmanagementtypes.Keycard{} kc.FromSyncKeycard(sKc) err = m.settings.SaveOrUpdateKeycard(kc, message.Clock, false) if err != nil { @@ -3721,7 +3682,7 @@ func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPa return nil, err } - if m.config.accountsFeed != nil { + if m.config.accountsPublisher != nil { addedAddresses := []gethcommon.Address{} removedAddresses := []gethcommon.Address{} if dbKeypair.Removed { @@ -3733,22 +3694,25 @@ func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPa if acc.Chat { continue } - if acc.Removed { + // We call the signals only if there is a change in the account list + // i.e. an account is unknown (not part of the Accounts list before) + // or an account was removed and is now back (Removed flag changed) + stillPresent, ok := oldAddresses[acc.Address] + if acc.Removed && (!ok || stillPresent) { removedAddresses = append(removedAddresses, gethcommon.Address(acc.Address)) - } else { + } else if !ok || !stillPresent { addedAddresses = append(addedAddresses, gethcommon.Address(acc.Address)) } } } + if len(addedAddresses) > 0 { - m.config.accountsFeed.Send(accountsevent.Event{ - Type: accountsevent.EventTypeAdded, + pubsub.Publish(m.config.accountsPublisher, accountsevent.AccountsAddedEvent{ Accounts: addedAddresses, }) } if len(removedAddresses) > 0 { - m.config.accountsFeed.Send(accountsevent.Event{ - Type: accountsevent.EventTypeRemoved, + pubsub.Publish(m.config.accountsPublisher, accountsevent.AccountsRemovedEvent{ Accounts: removedAddresses, }) } @@ -3757,7 +3721,7 @@ func (m *Messenger) handleSyncKeypair(message *protobuf.SyncKeypair, fromLocalPa return dbKeypair, nil } -func (m *Messenger) HandleSyncAccountsPositions(state *ReceivedMessageState, message *protobuf.SyncAccountsPositions, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncAccountsPositions(state *ReceivedMessageState, message *protobuf.SyncAccountsPositions, statusMessage *messagingtypes.Message) error { accs, err := m.handleSyncAccountsPositions(message) if err != nil { if err == ErrTryingToApplyOldWalletAccountsOrder || @@ -3775,7 +3739,7 @@ func (m *Messenger) HandleSyncAccountsPositions(state *ReceivedMessageState, mes return nil } -func (m *Messenger) HandleSyncTokenPreferences(state *ReceivedMessageState, message *protobuf.SyncTokenPreferences, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncTokenPreferences(state *ReceivedMessageState, message *protobuf.SyncTokenPreferences, statusMessage *messagingtypes.Message) error { tokenPreferences, err := m.handleSyncTokenPreferences(message) if err != nil { if err == ErrTryingToApplyOldTokenPreferences { @@ -3790,7 +3754,7 @@ func (m *Messenger) HandleSyncTokenPreferences(state *ReceivedMessageState, mess return nil } -func (m *Messenger) HandleSyncCollectiblePreferences(state *ReceivedMessageState, message *protobuf.SyncCollectiblePreferences, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncCollectiblePreferences(state *ReceivedMessageState, message *protobuf.SyncCollectiblePreferences, statusMessage *messagingtypes.Message) error { collectiblePreferences, err := m.handleSyncCollectiblePreferences(message) if err != nil { if err == ErrTryingToApplyOldCollectiblePreferences { @@ -3805,8 +3769,8 @@ func (m *Messenger) HandleSyncCollectiblePreferences(state *ReceivedMessageState return nil } -func (m *Messenger) HandleSyncAccount(state *ReceivedMessageState, message *protobuf.SyncAccount, statusMessage *v1protocol.StatusMessage) error { - acc, err := m.handleSyncWatchOnlyAccount(message, false) +func (m *Messenger) HandleSyncAccount(state *ReceivedMessageState, message *protobuf.SyncAccount, statusMessage *messagingtypes.Message) error { + acc, err := m.handleSyncWatchOnlyAccount(message) if err != nil { if err == ErrTryingToStoreOldWalletAccount { return nil @@ -3819,7 +3783,7 @@ func (m *Messenger) HandleSyncAccount(state *ReceivedMessageState, message *prot return nil } -func (m *Messenger) HandleSyncKeypair(state *ReceivedMessageState, message *protobuf.SyncKeypair, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncKeypair(state *ReceivedMessageState, message *protobuf.SyncKeypair, statusMessage *messagingtypes.Message) error { return m.handleSyncKeypairInternal(state, message, false) } @@ -3860,7 +3824,7 @@ func (m *Messenger) handleSyncKeypairInternal(state *ReceivedMessageState, messa return nil } -func (m *Messenger) HandleSyncContactRequestDecision(state *ReceivedMessageState, message *protobuf.SyncContactRequestDecision, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncContactRequestDecision(state *ReceivedMessageState, message *protobuf.SyncContactRequestDecision, statusMessage *messagingtypes.Message) error { var err error var response *MessengerResponse @@ -3876,7 +3840,7 @@ func (m *Messenger) HandleSyncContactRequestDecision(state *ReceivedMessageState return state.Response.Merge(response) } -func (m *Messenger) HandlePushNotificationRegistration(state *ReceivedMessageState, encryptedRegistration []byte, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandlePushNotificationRegistration(state *ReceivedMessageState, encryptedRegistration []byte, statusMessage *messagingtypes.Message) error { if m.pushNotificationServer == nil { return nil } @@ -3885,7 +3849,7 @@ func (m *Messenger) HandlePushNotificationRegistration(state *ReceivedMessageSta return m.pushNotificationServer.HandlePushNotificationRegistration(publicKey, encryptedRegistration) } -func (m *Messenger) HandlePushNotificationResponse(state *ReceivedMessageState, message *protobuf.PushNotificationResponse, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandlePushNotificationResponse(state *ReceivedMessageState, message *protobuf.PushNotificationResponse, statusMessage *messagingtypes.Message) error { if m.pushNotificationClient == nil { return nil } @@ -3894,7 +3858,7 @@ func (m *Messenger) HandlePushNotificationResponse(state *ReceivedMessageState, return m.pushNotificationClient.HandlePushNotificationResponse(publicKey, message) } -func (m *Messenger) HandlePushNotificationRegistrationResponse(state *ReceivedMessageState, message *protobuf.PushNotificationRegistrationResponse, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandlePushNotificationRegistrationResponse(state *ReceivedMessageState, message *protobuf.PushNotificationRegistrationResponse, statusMessage *messagingtypes.Message) error { if m.pushNotificationClient == nil { return nil } @@ -3903,7 +3867,7 @@ func (m *Messenger) HandlePushNotificationRegistrationResponse(state *ReceivedMe return m.pushNotificationClient.HandlePushNotificationRegistrationResponse(publicKey, message) } -func (m *Messenger) HandlePushNotificationQuery(state *ReceivedMessageState, message *protobuf.PushNotificationQuery, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandlePushNotificationQuery(state *ReceivedMessageState, message *protobuf.PushNotificationQuery, statusMessage *messagingtypes.Message) error { if m.pushNotificationServer == nil { return nil } @@ -3912,7 +3876,7 @@ func (m *Messenger) HandlePushNotificationQuery(state *ReceivedMessageState, mes return m.pushNotificationServer.HandlePushNotificationQuery(publicKey, statusMessage.ApplicationLayer.ID, message) } -func (m *Messenger) HandlePushNotificationQueryResponse(state *ReceivedMessageState, message *protobuf.PushNotificationQueryResponse, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandlePushNotificationQueryResponse(state *ReceivedMessageState, message *protobuf.PushNotificationQueryResponse, statusMessage *messagingtypes.Message) error { if m.pushNotificationClient == nil { return nil } @@ -3921,7 +3885,7 @@ func (m *Messenger) HandlePushNotificationQueryResponse(state *ReceivedMessageSt return m.pushNotificationClient.HandlePushNotificationQueryResponse(publicKey, message) } -func (m *Messenger) HandlePushNotificationRequest(state *ReceivedMessageState, message *protobuf.PushNotificationRequest, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandlePushNotificationRequest(state *ReceivedMessageState, message *protobuf.PushNotificationRequest, statusMessage *messagingtypes.Message) error { if m.pushNotificationServer == nil { return nil } @@ -3930,7 +3894,7 @@ func (m *Messenger) HandlePushNotificationRequest(state *ReceivedMessageState, m return m.pushNotificationServer.HandlePushNotificationRequest(publicKey, statusMessage.ApplicationLayer.ID, message) } -func (m *Messenger) HandleCommunityDescription(state *ReceivedMessageState, message *protobuf.CommunityDescription, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleCommunityDescription(state *ReceivedMessageState, message *protobuf.CommunityDescription, statusMessage *messagingtypes.Message) error { // shard passed as nil since it is handled within by using default shard err := m.handleCommunityDescription(state, state.CurrentMessageState.PublicKey, message, statusMessage.EncryptionLayer.Payload, nil, nil) if err != nil { @@ -3940,7 +3904,7 @@ func (m *Messenger) HandleCommunityDescription(state *ReceivedMessageState, mess return nil } -func (m *Messenger) HandleSyncBookmark(state *ReceivedMessageState, message *protobuf.SyncBookmark, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncBookmark(state *ReceivedMessageState, message *protobuf.SyncBookmark, statusMessage *messagingtypes.Message) error { bookmark := &browsers.Bookmark{ URL: message.Url, Name: message.Name, @@ -3952,7 +3916,7 @@ func (m *Messenger) HandleSyncBookmark(state *ReceivedMessageState, message *pro return nil } -func (m *Messenger) HandleSyncClearHistory(state *ReceivedMessageState, message *protobuf.SyncClearHistory, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncClearHistory(state *ReceivedMessageState, message *protobuf.SyncClearHistory, statusMessage *messagingtypes.Message) error { chatID := message.ChatId existingChat, ok := state.AllChats.Load(chatID) if !ok { @@ -3984,7 +3948,7 @@ func (m *Messenger) HandleSyncClearHistory(state *ReceivedMessageState, message return nil } -func (m *Messenger) HandleSyncTrustedUser(state *ReceivedMessageState, message *protobuf.SyncTrustedUser, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncTrustedUser(state *ReceivedMessageState, message *protobuf.SyncTrustedUser, statusMessage *messagingtypes.Message) error { updated, err := m.verificationDatabase.UpsertTrustStatus(message.Id, verification.TrustStatus(message.Status), message.Clock) if err != nil { return err @@ -4006,7 +3970,7 @@ func (m *Messenger) HandleSyncTrustedUser(state *ReceivedMessageState, message * return nil } -func (m *Messenger) HandleCommunityMessageArchiveMagnetlink(state *ReceivedMessageState, message *protobuf.CommunityMessageArchiveMagnetlink, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleCommunityMessageArchiveMagnetlink(state *ReceivedMessageState, message *protobuf.CommunityMessageArchiveMagnetlink, statusMessage *messagingtypes.Message) error { return m.HandleHistoryArchiveMagnetlinkMessage(state, state.CurrentMessageState.PublicKey, message.MagnetUri, message.Clock) } @@ -4038,7 +4002,7 @@ func (m *Messenger) addNewKeypairAddedOnPairedDeviceACNotification(keyUID string return nil } -func (m *Messenger) HandleSyncProfileShowcasePreferences(state *ReceivedMessageState, p *protobuf.SyncProfileShowcasePreferences, statusMessage *v1protocol.StatusMessage) error { +func (m *Messenger) HandleSyncProfileShowcasePreferences(state *ReceivedMessageState, p *protobuf.SyncProfileShowcasePreferences, statusMessage *messagingtypes.Message) error { _, err := m.saveProfileShowcasePreferencesProto(p, false) return err } diff --git a/protocol/messenger_local_backup.go b/protocol/messenger_local_backup.go index 26532c8ddbe..a5ece9e685e 100644 --- a/protocol/messenger_local_backup.go +++ b/protocol/messenger_local_backup.go @@ -2,24 +2,248 @@ package protocol import ( "errors" + "time" "github.com/golang/protobuf/proto" + "github.com/status-im/status-go/multiaccounts/settings" + "github.com/status-im/status-go/protocol/communities" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/signal" ) +const ( + BackupContactsPerBatch = 20 +) + +type CommunitySet struct { + Joined []*communities.Community + Deleted []*communities.Community +} + +func (m *Messenger) backupContacts() []*protobuf.Backup { + var contacts []*protobuf.SyncInstallationContactV2 + m.allContacts.Range(func(contactID string, contact *Contact) (shouldContinue bool) { + syncContact := m.buildSyncContactMessage(contact) + if syncContact != nil { + contacts = append(contacts, syncContact) + } + return true + }) + + var backupMessages []*protobuf.Backup + for i := 0; i < len(contacts); i += BackupContactsPerBatch { + j := min(i+BackupContactsPerBatch, len(contacts)) + + contactsToAdd := contacts[i:j] + + backupMessage := &protobuf.Backup{ + Contacts: contactsToAdd, + } + backupMessages = append(backupMessages, backupMessage) + } + + return backupMessages +} + +func (m *Messenger) retrieveAllCommunities() (*CommunitySet, error) { + joinedCs, err := m.communitiesManager.JoinedAndPendingCommunitiesWithRequests() + if err != nil { + return nil, err + } + + deletedCs, err := m.communitiesManager.DeletedCommunities() + if err != nil { + return nil, err + } + + return &CommunitySet{ + Joined: joinedCs, + Deleted: deletedCs, + }, nil +} + +func (m *Messenger) backupCommunity(community *communities.Community, clock uint64) (*protobuf.Backup, error) { + communityId := community.ID() + settings, err := m.communitiesManager.GetCommunitySettingsByID(communityId) + if err != nil { + return nil, err + } + + syncControlNode, err := m.communitiesManager.GetSyncControlNode(communityId) + if err != nil { + return nil, err + } + + syncMessage, err := community.ToSyncInstallationCommunityProtobuf(clock, settings, syncControlNode) + if err != nil { + return nil, err + } + + err = m.propagateSyncInstallationCommunityWithHRKeys(syncMessage, community) + if err != nil { + return nil, err + } + + return &protobuf.Backup{ + Communities: []*protobuf.SyncInstallationCommunity{syncMessage}, + }, nil +} + +func (m *Messenger) backupCommunities(clock uint64) ([]*protobuf.Backup, error) { + communitySet, err := m.retrieveAllCommunities() + if err != nil { + return nil, err + } + + var backupMessages []*protobuf.Backup + combinedCs := append(communitySet.Joined, communitySet.Deleted...) + + for _, c := range combinedCs { + _, beingImported := m.importingCommunities[c.IDString()] + if !beingImported { + backupMessage, err := m.backupCommunity(c, clock) + if err != nil { + return nil, err + } + + backupMessages = append(backupMessages, backupMessage) + } + } + + return backupMessages, nil +} + +func (m *Messenger) backupChats(clock uint64) []*protobuf.Backup { + var oneToOneAndGroupChats []*protobuf.SyncChat + m.allChats.Range(func(chatID string, chat *Chat) bool { + if !chat.OneToOne() && !chat.PrivateGroupChat() { + return true + } + syncChat := protobuf.SyncChat{ + Clock: clock, + Id: chatID, + ChatType: uint32(chat.ChatType), + Active: chat.Active, + } + chatMuteTill, _ := time.Parse(time.RFC3339, chat.MuteTill.Format(time.RFC3339)) + if chat.Muted && chatMuteTill.Equal(time.Time{}) { + // Only set Muted if it is "permanently" muted + syncChat.Muted = true + } + if chat.PrivateGroupChat() { + syncChat.Name = chat.Name // The Name is only useful in the case of a group chat + + syncChat.MembershipUpdateEvents = make([]*protobuf.MembershipUpdateEvents, len(chat.MembershipUpdates)) + for i, membershipUpdate := range chat.MembershipUpdates { + syncChat.MembershipUpdateEvents[i] = &protobuf.MembershipUpdateEvents{ + Clock: membershipUpdate.ClockValue, + Type: uint32(membershipUpdate.Type), + Members: membershipUpdate.Members, + Name: membershipUpdate.Name, + Signature: membershipUpdate.Signature, + ChatId: membershipUpdate.ChatID, + From: membershipUpdate.From, + RawPayload: membershipUpdate.RawPayload, + Color: membershipUpdate.Color, + Image: membershipUpdate.Image, + } + } + } + oneToOneAndGroupChats = append(oneToOneAndGroupChats, &syncChat) + return true + }) + + var backupMessages []*protobuf.Backup + backupMessage := &protobuf.Backup{ + Chats: oneToOneAndGroupChats, + } + backupMessages = append(backupMessages, backupMessage) + return backupMessages +} + +func (m *Messenger) backupProfile(clock uint64) (*protobuf.Backup, error) { + displayName, err := m.settings.DisplayName() + if err != nil { + return nil, err + } + + displayNameClock, err := m.settings.GetSettingLastSynced(settings.DisplayName) + if err != nil { + return nil, err + } + + if m.account == nil { + return nil, nil + } + + keyUID := m.account.KeyUID + images, err := m.multiAccounts.GetIdentityImages(keyUID) + if err != nil { + return nil, err + } + + pictureProtos := make([]*protobuf.SyncProfilePicture, len(images)) + for i, image := range images { + p := &protobuf.SyncProfilePicture{} + p.Name = image.Name + p.Payload = image.Payload + p.Width = uint32(image.Width) + p.Height = uint32(image.Height) + p.FileSize = uint32(image.FileSize) + p.ResizeTarget = uint32(image.ResizeTarget) + if image.Clock == 0 { + p.Clock = clock + } else { + p.Clock = image.Clock + } + pictureProtos[i] = p + } + + ensUsernameDetails, err := m.getEnsUsernameDetails() + if err != nil { + return nil, err + } + ensUsernameDetailProtos := make([]*protobuf.SyncEnsUsernameDetail, len(ensUsernameDetails)) + for i, ensUsernameDetail := range ensUsernameDetails { + ensUsernameDetailProtos[i] = &protobuf.SyncEnsUsernameDetail{ + Username: ensUsernameDetail.Username, + Clock: ensUsernameDetail.Clock, + Removed: ensUsernameDetail.Removed, + ChainId: ensUsernameDetail.ChainID, + } + } + + profileShowcasePreferences, err := m.GetProfileShowcasePreferences() + if err != nil { + return nil, err + } + + backupMessage := &protobuf.Backup{ + Profile: &protobuf.BackedUpProfile{ + KeyUid: keyUID, + DisplayName: displayName, + Pictures: pictureProtos, + DisplayNameClock: displayNameClock, + EnsUsernameDetails: ensUsernameDetailProtos, + ProfileShowcasePreferences: ToProfileShowcasePreferencesProto(profileShowcasePreferences), + }, + } + + return backupMessage, nil +} + func (m *Messenger) ExportBackup() ([]byte, error) { backup := &protobuf.MessengerLocalBackup{} clock, _ := m.getLastClockWithRelatedChat() - contactsToBackup := m.backupContacts(m.ctx) - communitiesToBackup, err := m.backupCommunities(m.ctx, clock) + contactsToBackup := m.backupContacts() + communitiesToBackup, err := m.backupCommunities(clock) if err != nil { return nil, err } - chatsToBackup := m.backupChats(m.ctx, clock) - profileToBackup, err := m.backupProfile(m.ctx, clock) + chatsToBackup := m.backupChats(clock) + profileToBackup, err := m.backupProfile(clock) if err != nil { return nil, err } diff --git a/protocol/messenger_mailserver.go b/protocol/messenger_mailserver.go index ecb11ba449c..63cbd56665e 100644 --- a/protocol/messenger_mailserver.go +++ b/protocol/messenger_mailserver.go @@ -14,8 +14,8 @@ import ( gocommon "github.com/status-im/status-go/common" "github.com/status-im/status-go/connection" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/services/mailservers" @@ -39,8 +39,9 @@ const ( var ErrNoFiltersForChat = errors.New("no filter registered for given chat") func (m *Messenger) shouldSync() (bool, error) { - // TODO (pablo) support community store node as well - if m.messaging.GetActiveStorenode().ID == "" || !m.Online() { + if !m.started || + m.messaging.GetActiveStorenode().ID == "" || + !m.Online() { return false, nil } @@ -66,7 +67,7 @@ func (m *Messenger) scheduleSyncChat(chat *Chat) (bool, error) { go func() { defer gocommon.LogOnPanic() - peerInfo := m.getCommunityStorenode(chat.CommunityID) + peerInfo := m.messaging.GetActiveStorenode() _, err = m.performStorenodeTask(func() (*MessengerResponse, error) { response, err := m.syncChatWithFilters(peerInfo, chat.ID) @@ -127,28 +128,23 @@ func (m *Messenger) scheduleSyncFilters(filters messagingtypes.ChatFilters) (boo go func() { defer gocommon.LogOnPanic() - // split filters by community store node so we can request the filters to the correct mailserver - filtersByMs := m.SplitFiltersByStoreNode(filters) - for communityID, filtersForMs := range filtersByMs { - peerInfo := m.getCommunityStorenode(communityID) - _, err := m.performStorenodeTask(func() (*MessengerResponse, error) { - response, err := m.syncFilters(peerInfo, filtersForMs) + peerInfo := m.messaging.GetActiveStorenode() + _, err := m.performStorenodeTask(func() (*MessengerResponse, error) { + response, err := m.syncFilters(peerInfo, filters) - if err != nil { - m.logger.Error("failed to sync filter", zap.Error(err)) - return nil, err - } - - if m.config.messengerSignalsHandler != nil { - m.config.messengerSignalsHandler.MessengerResponse(response) - } - return response, nil - }, history.WithPeerID(peerInfo.ID)) if err != nil { - m.logger.Error("failed to perform mailserver request", zap.Error(err)) + m.logger.Error("failed to sync filter", zap.Error(err)) + return nil, err } - } + if m.config.messengerSignalsHandler != nil { + m.config.messengerSignalsHandler.MessengerResponse(response) + } + return response, nil + }, history.WithPeerID(peerInfo.ID)) + if err != nil { + m.logger.Error("failed to perform mailserver request", zap.Error(err)) + } }() return true, nil } @@ -220,31 +216,6 @@ func (m *Messenger) syncChatWithFilters(peerInfo peer.AddrInfo, chatID string) ( return m.syncFilters(peerInfo, filters) } -func (m *Messenger) syncBackup() error { - - filter := m.messaging.PersonalTopicFilter() - if filter == nil { - return errors.New("personal topic filter not loaded") - } - canSync, err := m.canSyncWithStoreNodes() - if err != nil { - return err - } - if !canSync { - return nil - } - - from, to := m.calculateMailserverTimeBounds(oneMonthDuration) - - batch := messagingtypes.StoreNodeBatch{From: from, To: to, Topics: []messagingtypes.ContentTopic{filter.ContentTopic()}} - ms := m.getCommunityStorenode(filter.ChatID()) - err = m.processMailserverBatch(ms, batch) - if err != nil { - return err - } - return m.settings.SetBackupFetched(true) -} - func (m *Messenger) defaultSyncPeriodFromNow() (uint32, error) { defaultSyncPeriod, err := m.settings.GetDefaultSyncPeriod() if err != nil { @@ -291,24 +262,8 @@ func (m *Messenger) resetFiltersPriority(filters messagingtypes.ChatFilters) err return nil } -func (m *Messenger) SplitFiltersByStoreNode(filters messagingtypes.ChatFilters) map[string]messagingtypes.ChatFilters { - // split filters by community store node so we can request the filters to the correct mailserver - filtersByMs := make(map[string]messagingtypes.ChatFilters, len(filters)) - for _, f := range filters { - communityID := "" // none by default - if chat, ok := m.allChats.Load(f.ChatID()); ok && chat.CommunityChat() && m.communityStorenodes.HasStorenodeSetup(chat.CommunityID) { - communityID = chat.CommunityID - } - if _, exists := filtersByMs[communityID]; !exists { - filtersByMs[communityID] = make(messagingtypes.ChatFilters, 0, len(filters)) - } - filtersByMs[communityID] = append(filtersByMs[communityID], f) - } - return filtersByMs -} - // RequestAllHistoricMessages requests all the historic messages for any topic -func (m *Messenger) RequestAllHistoricMessages(forceFetchingBackup, withRetries bool) (*MessengerResponse, error) { +func (m *Messenger) RequestAllHistoricMessages(withRetries bool) (*MessengerResponse, error) { shouldSync, err := m.shouldSync() if err != nil { return nil, err @@ -318,24 +273,11 @@ func (m *Messenger) RequestAllHistoricMessages(forceFetchingBackup, withRetries return nil, nil } - backupFetched, err := m.settings.BackupFetched() - if err != nil { - return nil, err - } - if m.mailserversDatabase == nil { return nil, nil } allResponses := &MessengerResponse{} - if forceFetchingBackup || !backupFetched { - m.logger.Info("fetching backup") - err := m.syncBackup() - if err != nil { - return nil, err - } - m.logger.Info("backup fetched") - } filters := m.messaging.ChatFilters() err = m.updateFiltersPriority(filters) @@ -349,23 +291,12 @@ func (m *Messenger) RequestAllHistoricMessages(forceFetchingBackup, withRetries } }() - filtersByMs := m.SplitFiltersByStoreNode(filters) - for communityID, filtersForMs := range filtersByMs { - peerInfo := m.getCommunityStorenode(communityID) - if withRetries { - response, err := m.performStorenodeTask(func() (*MessengerResponse, error) { - return m.syncFilters(peerInfo, filtersForMs) - }, history.WithPeerID(peerInfo.ID)) - if err != nil { - return nil, err - } - if response != nil { - allResponses.AddChats(response.Chats()) - allResponses.AddMessages(response.Messages()) - } - continue - } - response, err := m.syncFilters(peerInfo, filtersForMs) + peerInfo := m.messaging.GetActiveStorenode() + + if withRetries { + response, err := m.performStorenodeTask(func() (*MessengerResponse, error) { + return m.syncFilters(peerInfo, filters) + }, history.WithPeerID(peerInfo.ID)) if err != nil { return nil, err } @@ -373,6 +304,16 @@ func (m *Messenger) RequestAllHistoricMessages(forceFetchingBackup, withRetries allResponses.AddChats(response.Chats()) allResponses.AddMessages(response.Messages()) } + return allResponses, nil + } + + response, err := m.syncFilters(peerInfo, filters) + if err != nil { + return nil, err + } + if response != nil { + allResponses.AddChats(response.Chats()) + allResponses.AddMessages(response.Messages()) } return allResponses, nil } @@ -400,15 +341,8 @@ func (m *Messenger) checkForMissingMessagesLoop() { } filters := m.messaging.ChatFilters() - filtersByMs := m.SplitFiltersByStoreNode(filters) - for communityID, filtersForMs := range filtersByMs { - peerInfo := m.getCommunityStorenode(communityID) - if peerInfo.ID == "" { - continue - } - - m.messaging.SetCriteriaForMissingMessageVerification(peerInfo, filtersForMs) - } + peerInfo := m.messaging.GetActiveStorenode() + m.messaging.SetCriteriaForMissingMessageVerification(peerInfo, filters) } } @@ -699,7 +633,7 @@ func (m *Messenger) SyncChatFromSyncedFrom(chatID string) (uint32, error) { return 0, ErrChatNotFound } - peerInfo := m.getCommunityStorenode(chat.CommunityID) + peerInfo := m.messaging.GetActiveStorenode() var from uint32 _, err := m.performStorenodeTask(func() (*MessengerResponse, error) { canSync, err := m.canSyncWithStoreNodes() @@ -762,7 +696,7 @@ func (m *Messenger) FillGaps(chatID string, messageIDs []string) error { return err } - chat, ok := m.allChats.Load(chatID) + _, ok := m.allChats.Load(chatID) if !ok { return errors.New("chat not existing") } @@ -800,7 +734,7 @@ func (m *Messenger) FillGaps(chatID string, messageIDs []string) error { m.config.messengerSignalsHandler.HistoryRequestStarted(1) } - peerID := m.getCommunityStorenode(chat.CommunityID) + peerID := m.messaging.GetActiveStorenode() err = m.processMailserverBatch(peerID, batch) if err != nil { return err @@ -841,17 +775,6 @@ func (m *Messenger) RemoveFilters(filters []*messagingtypes.ChatFilter) error { func (m *Messenger) ConnectionChanged(state connection.State) { m.messaging.ConnectionChanged(state) - if !m.connectionState.Offline && state.Offline { - m.sender.StopDatasync() - } - - if m.connectionState.Offline && !state.Offline { - err := m.sender.StartDatasync(m.mvdsStatusChangeEvent, m.sendDataSync) - if err != nil { - m.logger.Error("failed to start datasync", zap.Error(err)) - } - } - m.connectionState = state } @@ -863,7 +786,7 @@ func (m *Messenger) fetchMessages(chatID string, duration time.Duration) (uint32 return 0, ErrChatNotFound } - peerInfo := m.getCommunityStorenode(chat.CommunityID) + peerInfo := m.messaging.GetActiveStorenode() _, err := m.performStorenodeTask(func() (*MessengerResponse, error) { canSync, err := m.canSyncWithStoreNodes() if err != nil { diff --git a/protocol/messenger_mailserver_cycle.go b/protocol/messenger_mailserver_cycle.go index 65aa6053e5e..01ed4a2b264 100644 --- a/protocol/messenger_mailserver_cycle.go +++ b/protocol/messenger_mailserver_cycle.go @@ -66,7 +66,7 @@ func (m *Messenger) asyncRequestAllHistoricMessages() { go func() { defer gocommon.LogOnPanic() - _, err := m.RequestAllHistoricMessages(false, true) + _, err := m.RequestAllHistoricMessages(true) if err != nil { m.logger.Error("failed to request historic messages", zap.Error(err)) } diff --git a/protocol/messenger_sync_contact.go b/protocol/messenger_sync_contact.go new file mode 100644 index 00000000000..cab192e4b07 --- /dev/null +++ b/protocol/messenger_sync_contact.go @@ -0,0 +1,45 @@ +package protocol + +import ( + multiaccountscommon "github.com/status-im/status-go/multiaccounts/common" + "github.com/status-im/status-go/protocol/protobuf" +) + +func (m *Messenger) buildSyncContactMessage(contact *Contact) *protobuf.SyncInstallationContactV2 { + var ensName string + if contact.ENSVerified { + ensName = contact.EnsName + } + + var customizationColor uint32 + if len(contact.CustomizationColor) != 0 { + customizationColor = multiaccountscommon.ColorToIDFallbackToBlue(contact.CustomizationColor) + } + + oneToOneChat, ok := m.allChats.Load(contact.ID) + muted := false + if ok { + muted = oneToOneChat.Muted + } + + return &protobuf.SyncInstallationContactV2{ + LastUpdatedLocally: contact.LastUpdatedLocally, + LastUpdated: contact.LastUpdated, + Id: contact.ID, + DisplayName: contact.DisplayName, + EnsName: ensName, + CustomizationColor: customizationColor, + LocalNickname: contact.LocalNickname, + Added: contact.added(), + Blocked: contact.Blocked, + Muted: muted, + HasAddedUs: contact.hasAddedUs(), + Removed: contact.Removed, + ContactRequestLocalState: int64(contact.ContactRequestLocalState), + ContactRequestRemoteState: int64(contact.ContactRequestRemoteState), + ContactRequestRemoteClock: int64(contact.ContactRequestRemoteClock), + ContactRequestLocalClock: int64(contact.ContactRequestLocalClock), + VerificationStatus: int64(contact.VerificationStatus), + TrustStatus: int64(contact.TrustStatus), + } +} diff --git a/protocol/messenger_testing_utils.go b/protocol/messenger_testing_utils.go index ece3da36c9f..e4ea487dccb 100644 --- a/protocol/messenger_testing_utils.go +++ b/protocol/messenger_testing_utils.go @@ -69,7 +69,6 @@ func (m *MessengerSignalsHandlerMock) SendWakuBackedUpKeypair(*wakusync.WakuBack func (m *MessengerSignalsHandlerMock) SendWakuBackedUpWatchOnlyAccount(*wakusync.WakuBackedUpDataResponse) { } -func (m *MessengerSignalsHandlerMock) BackupPerformed(uint64) {} func (m *MessengerSignalsHandlerMock) HistoryArchivesProtocolEnabled() {} func (m *MessengerSignalsHandlerMock) HistoryArchivesProtocolDisabled() {} func (m *MessengerSignalsHandlerMock) CreatingHistoryArchives(string) {} diff --git a/protocol/protobuf/application_metadata_message.proto b/protocol/protobuf/application_metadata_message.proto index 4ff8a157673..df56975af18 100644 --- a/protocol/protobuf/application_metadata_message.proto +++ b/protocol/protobuf/application_metadata_message.proto @@ -50,7 +50,7 @@ message ApplicationMetadataMessage { ANONYMOUS_METRIC_BATCH = 33; SYNC_CHAT_REMOVED = 34; SYNC_CHAT_MESSAGES_READ = 35; - BACKUP = 36; + BACKUP = 36 [deprecated=true]; SYNC_ACTIVITY_CENTER_READ = 37; SYNC_ACTIVITY_CENTER_ACCEPTED = 38; SYNC_ACTIVITY_CENTER_DISMISSED = 39; diff --git a/protocol/protobuf/sync_settings.proto b/protocol/protobuf/sync_settings.proto index 5ea96402522..6895f28c596 100644 --- a/protocol/protobuf/sync_settings.proto +++ b/protocol/protobuf/sync_settings.proto @@ -42,8 +42,3 @@ message SyncSetting { AUTO_REFRESH_TOKENS_ENABLED = 22; } } - -/* TODOs -LastBackup uint64 -BackupEnabled bool - */ diff --git a/protocol/requests/restore_account.go b/protocol/requests/restore_account.go index fd0be680a1d..75a7f928e0e 100644 --- a/protocol/requests/restore_account.go +++ b/protocol/requests/restore_account.go @@ -16,8 +16,6 @@ type RestoreAccount struct { // This is to log in using a keycard with existing account. Keycard *KeycardData `json:"keycard"` - FetchBackup bool `json:"fetchBackup"` - CreateAccount } diff --git a/server/pairing/sync_device_test.go b/server/pairing/sync_device_test.go index a931f06739c..faf5831263f 100644 --- a/server/pairing/sync_device_test.go +++ b/server/pairing/sync_device_test.go @@ -130,7 +130,6 @@ func (s *SyncDeviceSuite) prepareBackendWithAccount(mnemonic, tmpdir string) *ap } else { _, err = backend.RestoreAccountAndLogin(&requests.RestoreAccount{ Mnemonic: mnemonic, - FetchBackup: false, CreateAccount: createAccount, }, opts...) } diff --git a/services/ext/api.go b/services/ext/api.go index dcfd8ff665a..90c99aca878 100644 --- a/services/ext/api.go +++ b/services/ext/api.go @@ -1371,12 +1371,12 @@ func (api *PublicAPI) ToggleNewsRSSEnabled(value bool) error { return m.ToggleNewsRSSEnabled(value) } -func (api *PublicAPI) RequestAllHistoricMessages(forceFetchingBackup bool) (*protocol.MessengerResponse, error) { - return api.service.messenger.RequestAllHistoricMessages(forceFetchingBackup, false) +func (api *PublicAPI) RequestAllHistoricMessages() (*protocol.MessengerResponse, error) { + return api.service.messenger.RequestAllHistoricMessages(false) } -func (api *PublicAPI) RequestAllHistoricMessagesWithRetries(forceFetchingBackup bool) (*protocol.MessengerResponse, error) { - return api.service.messenger.RequestAllHistoricMessages(forceFetchingBackup, true) +func (api *PublicAPI) RequestAllHistoricMessagesWithRetries() (*protocol.MessengerResponse, error) { + return api.service.messenger.RequestAllHistoricMessages(true) } // Echo is a method for testing purposes. @@ -1492,10 +1492,6 @@ func (api *PublicAPI) ChangeIdentityImageShowTo(showTo settings.ProfilePicturesS return api.service.messenger.PublishIdentityImage() } -func (api *PublicAPI) BackupData() (uint64, error) { - return api.service.messenger.BackupData(context.Background()) -} - func (api *PublicAPI) ImageServerURL() string { return api.service.messenger.ImageServerURL() } diff --git a/services/ext/service.go b/services/ext/service.go index ea8db50683e..4cc69c05526 100644 --- a/services/ext/service.go +++ b/services/ext/service.go @@ -153,9 +153,6 @@ func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, appD return err } s.messenger = messenger - if s.config.ProcessBackedupMessages { - s.messenger.EnableBackedupMessagesProcessing() - } // Be mindful of adding more initialization code, as it can easily // impact login times for mobile users. For example, we avoid calling diff --git a/services/ext/signal.go b/services/ext/signal.go index fcb2163f574..baaeb468a9c 100644 --- a/services/ext/signal.go +++ b/services/ext/signal.go @@ -61,11 +61,6 @@ func (m *MessengerSignalsHandler) MessageDelivered(chatID string, messageID stri signal.SendMessageDelivered(chatID, messageID) } -// BackupPerformed passes information that a backup was performed -func (m *MessengerSignalsHandler) BackupPerformed(lastBackup uint64) { - signal.SendBackupPerformed(lastBackup) -} - // CommunityInfoFound passes info about community that was requested before func (m *MessengerSignalsHandler) CommunityInfoFound(community *communities.Community) { signal.SendCommunityInfoFound(community) diff --git a/signal/events_shhext.go b/signal/events_shhext.go index 61632100002..390cb6a7574 100644 --- a/signal/events_shhext.go +++ b/signal/events_shhext.go @@ -192,11 +192,6 @@ type EnodeDiscoveredSignal struct { Topic string `json:"topic"` } -// BackupPerformedSignal signals that a backup has been performed -type BackupPerformedSignal struct { - LastBackup uint64 `json:"lastBackup"` -} - // SendEnodeDiscovered tiggered when an enode is discovered. // finds a new enode. func SendEnodeDiscovered(enode, topic string) { @@ -206,9 +201,6 @@ func SendEnodeDiscovered(enode, topic string) { }) } -func SendBackupPerformed(lastBackup uint64) { - send(EventBackupPerformed, BackupPerformedSignal{lastBackup}) -} func SendDecryptMessageFailed(sender string) { send(EventDecryptMessageFailed, DecryptMessageFailedSignal{sender}) } diff --git a/tests-functional/resources/test_data/mnemonic_12.py b/tests-functional/resources/test_data/mnemonic_12.py index aaba1ae739e..555478499be 100644 --- a/tests-functional/resources/test_data/mnemonic_12.py +++ b/tests-functional/resources/test_data/mnemonic_12.py @@ -52,7 +52,6 @@ "use-mailservers?": True, "wallet-root-address": "0x1846a7930d0ab03e5a120ebdd46eff3fe0365824", "send-status-updates?": True, - "backup-enabled?": True, "show-community-asset-when-sending-tokens?": True, "display-assets-below-balance-threshold": 100000000, "url-unfurling-mode": 1, From 5a1d8ef34e1a9e1679c754800c23717a7aadafb1 Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Thu, 28 Aug 2025 12:02:09 -0400 Subject: [PATCH 09/15] fix(backup): use profile keypair for backup files Fixes https://github.com/status-im/status-desktop/issues/18731 Turns out our pubkeys all start with the same prefix, and I chose exactly the length of the prefix in the previous version, so all files had the same name. Using the profile keypair solves this --- node/get_status_node.go | 21 ++++++++++++--------- services/accounts/service.go | 5 ++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/node/get_status_node.go b/node/get_status_node.go index 628d2306d4a..13624600451 100644 --- a/node/get_status_node.go +++ b/node/get_status_node.go @@ -24,7 +24,6 @@ import ( "github.com/status-im/status-go/node/backup" "github.com/status-im/status-go/params" "github.com/status-im/status-go/pkg/pubsub" - "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/rpc" "github.com/status-im/status-go/server" accountssvc "github.com/status-im/status-go/services/accounts" @@ -231,14 +230,11 @@ func (n *StatusNode) StartLocalBackup() error { return errors.New("local backup already started") } - chatAccount, err := n.gethAccountsManager.SelectedChatAccount() - if err != nil { - return err - } - - privateKey := chatAccount.PrivateKey() filenameGetter := func() (string, error) { - accountIdentifier := common.PubkeyToHex(&privateKey.PublicKey) + profileKeypair, err := n.accountsSrvc.GetProfileKeypair() + if err != nil { + return "", err + } backupPath, err := n.accountsSrvc.GetBackupPath() if err != nil { @@ -250,10 +246,17 @@ func (n *StatusNode) StartLocalBackup() error { } else { backupDir = filepath.Join(n.config.RootDataDir, "backups") } - fullPath := filepath.Join(backupDir, fmt.Sprintf("%x_user_data.bkp", accountIdentifier[:4])) + fullPath := filepath.Join(backupDir, fmt.Sprintf("%x_user_data.bkp", profileKeypair.KeyUID[len(profileKeypair.KeyUID)-4:])) return fullPath, nil } + chatAccount, err := n.gethAccountsManager.SelectedChatAccount() + if err != nil { + return err + } + + privateKey := chatAccount.PrivateKey() + n.localBackup, err = backup.NewController(backup.BackupConfig{ PrivateKey: crypto.Keccak256(crypto.FromECDSA(privateKey)), FileNameGetter: filenameGetter, diff --git a/services/accounts/service.go b/services/accounts/service.go index 514b0479be0..b1d4a62c136 100644 --- a/services/accounts/service.go +++ b/services/accounts/service.go @@ -76,8 +76,11 @@ func (s *Service) AccountsAPI() *API { return NewAccountsAPI(s.manager, s.config, s.db, s.feed, &s.messenger) } -func (s *Service) GetKeypairByKeyUID(keyUID string) (*accounts.Keypair, error) { +func (s *Service) GetProfileKeypair() (*accounts.Keypair, error) { + return s.db.GetProfileKeypair() +} +func (s *Service) GetKeypairByKeyUID(keyUID string) (*accounts.Keypair, error) { return s.db.GetKeypairByKeyUID(keyUID) } From daffcd432526c1bec1566acf9b64a14f0b77b05b Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Wed, 27 Aug 2025 10:34:41 -0400 Subject: [PATCH 10/15] feat(backup): set backup path to user's config folder if not set This should make it work on mobile as well plus it sets a default to the setting --- node/get_status_node.go | 18 ++++++++++++++++++ services/accounts/service.go | 4 ++++ tests-functional/tests/test_local_backup.py | 3 +++ 3 files changed, 25 insertions(+) diff --git a/node/get_status_node.go b/node/get_status_node.go index 13624600451..9f2757907b7 100644 --- a/node/get_status_node.go +++ b/node/get_status_node.go @@ -230,6 +230,24 @@ func (n *StatusNode) StartLocalBackup() error { return errors.New("local backup already started") } + backupPath, err := n.accountsSrvc.GetBackupPath() + if err != nil { + return err + } + if backupPath == "" { + // No path set yet, set it to the user's config directory + dir, err := os.UserConfigDir() + // We do not return the error as it's not a major issue + if err != nil { + n.logger.Error("failed to get user config dir", zap.Error(err)) + } else { + err = n.accountsSrvc.SetBackupPath(filepath.Join(dir, "Status", "backups")) + if err != nil { + n.logger.Error("failed to set backup path", zap.Error(err)) + } + } + } + filenameGetter := func() (string, error) { profileKeypair, err := n.accountsSrvc.GetProfileKeypair() if err != nil { diff --git a/services/accounts/service.go b/services/accounts/service.go index b1d4a62c136..47d7a8fafde 100644 --- a/services/accounts/service.go +++ b/services/accounts/service.go @@ -92,6 +92,10 @@ func (s *Service) GetBackupPath() (string, error) { return s.db.BackupPath() } +func (s *Service) SetBackupPath(path string) error { + return s.db.SaveSettingField(settings.BackupPath, path) +} + func (s *Service) GetMessenger() *protocol.Messenger { return s.messenger } diff --git a/tests-functional/tests/test_local_backup.py b/tests-functional/tests/test_local_backup.py index 6eadc827210..ce3f0b9967e 100644 --- a/tests-functional/tests/test_local_backup.py +++ b/tests-functional/tests/test_local_backup.py @@ -246,6 +246,9 @@ def test_local_backup(self, tmp_path): assert group_chat_recovered, "Group chat was not restored correctly" assert one_on_one_chat_recovered, "One-to-one chat was not restored correctly" + # Change the backup path (Docker restricts access to a lot of folders) + backend_client.rpc_valid_request("settings_saveSetting", ["backup-path", "/usr/status-user/backups"]) + # Perform the backup response = backend_client.api_valid_request("PerformLocalBackup", "") jsonResponse = response.json() From 197dfc3e59176d5dc45ca4b6a81740eee06d322a Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Wed, 3 Sep 2025 12:03:33 -0400 Subject: [PATCH 11/15] fix(backup)_: use compressed pubkey for the local backup name It is the name that users are used to see, so it will be less confusing --- node/get_status_node.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/node/get_status_node.go b/node/get_status_node.go index 9f2757907b7..011d89578df 100644 --- a/node/get_status_node.go +++ b/node/get_status_node.go @@ -46,6 +46,7 @@ import ( "github.com/status-im/status-go/services/stickers" "github.com/status-im/status-go/services/subscriptions" "github.com/status-im/status-go/services/updates" + "github.com/status-im/status-go/services/utils" "github.com/status-im/status-go/services/wakuv2ext" "github.com/status-im/status-go/services/wallet" "github.com/status-im/status-go/services/web3provider" @@ -248,32 +249,35 @@ func (n *StatusNode) StartLocalBackup() error { } } + chatAccount, err := n.gethAccountsManager.SelectedChatAccount() + if err != nil { + return err + } + + privateKey := chatAccount.PrivateKey() + filenameGetter := func() (string, error) { - profileKeypair, err := n.accountsSrvc.GetProfileKeypair() + backupPath, err := n.accountsSrvc.GetBackupPath() if err != nil { return "", err } - backupPath, err := n.accountsSrvc.GetBackupPath() + compressedPubKey, err := utils.SerializePublicKey(crypto.CompressPubkey(&privateKey.PublicKey)) if err != nil { return "", err } + var backupDir string if backupPath != "" { backupDir = backupPath } else { backupDir = filepath.Join(n.config.RootDataDir, "backups") } - fullPath := filepath.Join(backupDir, fmt.Sprintf("%x_user_data.bkp", profileKeypair.KeyUID[len(profileKeypair.KeyUID)-4:])) - return fullPath, nil - } - chatAccount, err := n.gethAccountsManager.SelectedChatAccount() - if err != nil { - return err - } + fullPath := filepath.Join(backupDir, fmt.Sprintf("%s_user_data.bkp", compressedPubKey[len(compressedPubKey)-6:])) - privateKey := chatAccount.PrivateKey() + return fullPath, nil + } n.localBackup, err = backup.NewController(backup.BackupConfig{ PrivateKey: crypto.Keccak256(crypto.FromECDSA(privateKey)), From 33f84c7ab1f4fc1ce290e47ba9c1dfd3f94f5dba Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Thu, 28 Aug 2025 16:34:55 -0400 Subject: [PATCH 12/15] fix(backup): backup import fails if the community was left This is a fix for the backup and also just a performance improvement. Basically, when we have a SyncCommunity message, we leave the community if the state of said community is not pending or joined (meaning we left it). However, in the case of backup messages, we backup left communities too, but when trying to "leave" those communities, it threw an error about unknown chat, since we never spectated that community in that imported profile. The solution is just to early return when we don't have anything to leave. We gracefully ignore the error in the Sync function and pass the error when actively trying to leave (you can't leave twice). Added tests and also cleaned up some warnings. --- protocol/communities/manager.go | 6 +++ protocol/communities_messenger_test.go | 58 ++++++++++++++++++++++++++ protocol/messenger_communities.go | 24 +++++------ 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/protocol/communities/manager.go b/protocol/communities/manager.go index 9c3dbd41f2c..1c679893e9d 100644 --- a/protocol/communities/manager.go +++ b/protocol/communities/manager.go @@ -85,6 +85,7 @@ var ( ErrTorrentTimedout = errors.New("torrent has timed out") ErrCommunityRequestAlreadyRejected = errors.New("that user was already rejected from the community") ErrInvalidClock = errors.New("invalid clock to cancel request to join") + ErrNotPartOfCommunity = errors.New("not part of the community") ) type Manager struct { @@ -3734,6 +3735,11 @@ func (m *Manager) LeaveCommunity(id types.HexBytes) (*Community, error) { return nil, err } + if !community.Joined() && !community.Spectated() { + // If we are not joined or spectating, there is nothing to leave + return nil, ErrNotPartOfCommunity + } + community.RemoveOurselvesFromOrg(&m.identity.PublicKey) community.Leave() diff --git a/protocol/communities_messenger_test.go b/protocol/communities_messenger_test.go index 0157ccbd544..f65cb51aa7e 100644 --- a/protocol/communities_messenger_test.go +++ b/protocol/communities_messenger_test.go @@ -2162,6 +2162,24 @@ func (s *MessengerCommunitiesSuite) TestLeaveAndRejoinCommunity() { s.Require().Equal(1, numberInactiveChats) } +func (s *MessengerCommunitiesSuite) TestLeaveCommunityTwice() { + community, _ := s.createCommunity() + advertiseCommunityToUserOldWay(&s.Suite, community, s.owner, s.alice) + + _, err := s.alice.SpectateCommunity(community.ID()) + s.Require().NoError(err) + + response, err := s.alice.LeaveCommunity(community.ID()) + s.Require().NoError(err) + s.Require().NotNil(response) + + // Try to leave again + response, err = s.alice.LeaveCommunity(community.ID()) + s.Require().Error(err) + s.Require().Equal(communities.ErrNotPartOfCommunity, err) + s.Require().Nil(response) +} + func (s *MessengerCommunitiesSuite) TestShareCommunity() { description := &requests.CreateCommunity{ Membership: protobuf.CommunityPermissions_MANUAL_ACCEPT, @@ -3145,6 +3163,46 @@ func (s *MessengerCommunitiesSuite) TestSyncCommunity_OutdatedDescription() { s.Require().False(community.Spectated()) } +func (s *MessengerCommunitiesSuite) TestSyncCommunity_LeftCommunity() { + community, _ := s.createCommunity() + s.advertiseCommunityTo(community, s.owner, s.alice) + + // Join community + s.joinCommunity(community, s.owner, s.alice) + + // Update owner's community reference + community, err := s.owner.GetCommunityByID(community.ID()) + s.Require().NoError(err) + s.Require().True(community.HasMember(s.alice.IdentityPublicKey())) + + // Leave community + response, err := s.alice.LeaveCommunity(community.ID()) + s.Require().NoError(err) + s.Require().NotNil(response) + s.Require().False(response.Communities()[0].Joined()) + + // Create another device + aliceOtherDevice := s.createOtherDevice(s.alice) + defer TearDownMessenger(&s.Suite, aliceOtherDevice) + + // Create sync message + syncCommunityMsg, err := s.alice.buildSyncInstallationCommunity(response.Communities()[0], 1) + s.Require().NoError(err) + s.Require().False(syncCommunityMsg.Joined) + s.Require().False(syncCommunityMsg.Spectated) + + // Then make other device handle sync message with outdated community description + messageState := aliceOtherDevice.buildMessageState() + err = aliceOtherDevice.handleSyncInstallationCommunity(messageState, syncCommunityMsg) + s.Require().NoError(err) + + // Then community should not be joined + community, err = aliceOtherDevice.communitiesManager.GetByID(community.ID()) + s.Require().NoError(err) + s.Require().False(community.Joined()) + s.Require().False(community.Spectated()) +} + func (s *MessengerCommunitiesSuite) TestSetMutePropertyOnChatsByCategory() { // Create a community createCommunityReq := &requests.CreateCommunity{ diff --git a/protocol/messenger_communities.go b/protocol/messenger_communities.go index 55fc35851eb..c7fb97ca5fd 100644 --- a/protocol/messenger_communities.go +++ b/protocol/messenger_communities.go @@ -76,15 +76,15 @@ const ( ) const ( - ErrOwnerTokenNeeded = "Owner token is needed" // #nosec G101 - ErrMissingCommunityID = "CommunityID has to be provided" - ErrForbiddenProfileOrWatchOnlyAccount = "Cannot join a community using profile chat or watch-only account" - ErrSigningJoinRequestForKeycardAccounts = "Signing a joining community request for accounts migrated to keycard must be done with a keycard" - ErrNotPartOfCommunity = "Not part of the community" - ErrNotAdminOrOwner = "Not admin or owner" - ErrSignerIsNil = "Signer can't be nil" - ErrSyncMessagesSentByNonControlNode = "Accepted/requested to join sync messages can be send only by the control node" - ErrReceiverIsNil = "Receiver can't be nil" + ErrOwnerTokenNeeded = "owner token is needed" // #nosec G101 + ErrMissingCommunityID = "communityID has to be provided" + ErrForbiddenProfileOrWatchOnlyAccount = "cannot join a community using profile chat or watch-only account" + ErrSigningJoinRequestForKeycardAccounts = "signing a joining community request for accounts migrated to keycard must be done with a keycard" + ErrNotPartOfCommunity = "not part of the community" + ErrNotAdminOrOwner = "not admin or owner" + ErrSignerIsNil = "signer can't be nil" + ErrSyncMessagesSentByNonControlNode = "accepted/requested to join sync messages can be send only by the control node" + ErrReceiverIsNil = "receiver can't be nil" ) type FetchCommunityRequest struct { @@ -2969,10 +2969,6 @@ func (m *Messenger) ImportCommunity(ctx context.Context, key *ecdsa.PrivateKey) return nil, err } - if err != nil { - return nil, err - } - _, err = m.FetchCommunity(&FetchCommunityRequest{ CommunityKey: community.IDString(), Shard: community.Shard(), @@ -3858,7 +3854,7 @@ func (m *Messenger) handleSyncInstallationCommunity(messageState *ReceivedMessag } } else { mr, err = m.leaveCommunity(syncCommunity.Id) - if err != nil { + if err != nil && err != communities.ErrNotPartOfCommunity { logger.Debug("m.leaveCommunity error", zap.Error(err)) return err } From 1d3fdc71529e10f48b784a9dc2ebc9a3af6aee0b Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Thu, 7 Aug 2025 15:02:01 -0400 Subject: [PATCH 13/15] feat(backup): implement messages backup poc Fixes https://github.com/status-im/status-desktop/issues/18529 Implements a rough but functional proof of concept for message backup and import. It is guarded by a new setting so that you need to opt-in to have it, thus not breaking PFS by default. It works for text messages, contacts requests, images, emojis, transaction commands, pins and stickers. --- ...7511667_add_messages-backup_enabled.up.sql | 1 + multiaccounts/settings/columns.go | 6 + multiaccounts/settings/database.go | 25 +- .../settings/database_settings_manager.go | 1 + multiaccounts/settings/database_test.go | 46 +++ multiaccounts/settings/structs.go | 1 + protocol/common/message_test.go | 6 +- protocol/message_persistence.go | 341 ++++++++++++++++++ protocol/messenger_backup_handler.go | 21 +- protocol/messenger_local_backup.go | 20 + protocol/messenger_local_backup_test.go | 146 +++++++- protocol/messenger_test.go | 4 +- .../protobuf/messenger_local_backup.proto | 47 ++- services/accounts/settings.go | 4 + 14 files changed, 653 insertions(+), 16 deletions(-) create mode 100644 appdatabase/migrations/sql/1757511667_add_messages-backup_enabled.up.sql diff --git a/appdatabase/migrations/sql/1757511667_add_messages-backup_enabled.up.sql b/appdatabase/migrations/sql/1757511667_add_messages-backup_enabled.up.sql new file mode 100644 index 00000000000..554ae1b740f --- /dev/null +++ b/appdatabase/migrations/sql/1757511667_add_messages-backup_enabled.up.sql @@ -0,0 +1 @@ +ALTER TABLE settings ADD COLUMN messages_backup_enabled BOOLEAN DEFAULT FALSE; diff --git a/multiaccounts/settings/columns.go b/multiaccounts/settings/columns.go index a2261da44f7..7ec3ca2eedb 100644 --- a/multiaccounts/settings/columns.go +++ b/multiaccounts/settings/columns.go @@ -171,6 +171,11 @@ var ( reactFieldName: "log-level", dBColumnName: "log_level", } + MessagesBackupEnabled = SettingField{ + reactFieldName: "messages-backup-enabled", + dBColumnName: "messages_backup_enabled", + valueHandler: BoolHandler, + } MessagesFromContactsOnly = SettingField{ reactFieldName: "messages-from-contacts-only", dBColumnName: "messages_from_contacts_only", @@ -565,6 +570,7 @@ var ( LinkPreviewRequestEnabled, LinkPreviewsEnabledSites, LogLevel, + MessagesBackupEnabled, MessagesFromContactsOnly, Mnemonic, MnemonicRemoved, diff --git a/multiaccounts/settings/database.go b/multiaccounts/settings/database.go index 64d672012a8..641decbd2a3 100644 --- a/multiaccounts/settings/database.go +++ b/multiaccounts/settings/database.go @@ -406,7 +406,8 @@ func (db *Database) GetSettings() (Settings, error) { test_networks_enabled, mutual_contact_enabled, profile_migration_needed, wallet_token_preferences_group_by_community, url_unfurling_mode, mnemonic_was_not_shown, wallet_show_community_asset_when_sending_tokens, wallet_display_assets_below_balance, wallet_display_assets_below_balance_threshold, wallet_collectible_preferences_group_by_collection, wallet_collectible_preferences_group_by_community, - peer_syncing_enabled, auto_refresh_tokens_enabled, last_tokens_update, news_feed_enabled, news_feed_last_fetched_timestamp, news_rss_enabled, backup_path + peer_syncing_enabled, auto_refresh_tokens_enabled, last_tokens_update, news_feed_enabled, news_feed_last_fetched_timestamp, news_rss_enabled, backup_path, + thirdparty_services_enabled, messages_backup_enabled FROM settings WHERE @@ -495,6 +496,8 @@ func (db *Database) GetSettings() (Settings, error) { &newsFeedLastFetchedTimestamp, &s.NewsRSSEnabled, &s.BackupPath, + &s.ThirdpartyServicesEnabled, + &s.MessagesBackupEnabled, ) if err != nil { @@ -912,3 +915,23 @@ func (db *Database) BackupPath() (result string, err error) { } return result, err } + +func (db *Database) SetThirdpartyServicesEnabled(enabled bool) error { + return db.SaveSettingField(ThirdpartyServicesEnabled, enabled) +} + +func (db *Database) ThirdpartyServicesEnabled() (result bool, err error) { + err = db.makeSelectRow(ThirdpartyServicesEnabled).Scan(&result) + if err == sql.ErrNoRows { + return true, nil + } + return result, err +} + +func (db *Database) MessagesBackupEnabled() (result bool, err error) { + err = db.makeSelectRow(MessagesBackupEnabled).Scan(&result) + if err == sql.ErrNoRows { + return result, nil + } + return result, err +} diff --git a/multiaccounts/settings/database_settings_manager.go b/multiaccounts/settings/database_settings_manager.go index 8365ca97d97..ad410adc9e6 100644 --- a/multiaccounts/settings/database_settings_manager.go +++ b/multiaccounts/settings/database_settings_manager.go @@ -61,6 +61,7 @@ type DatabaseSettingsManager interface { CanSyncOnMobileNetwork() (result bool, err error) ShouldBroadcastUserStatus() (result bool, err error) BackupPath() (result string, err error) + MessagesBackupEnabled() (result bool, err error) AutoMessageEnabled() (result bool, err error) ENSName() (string, error) DeviceName() (string, error) diff --git a/multiaccounts/settings/database_test.go b/multiaccounts/settings/database_test.go index 2b8ee2d258a..460e6d3fb94 100644 --- a/multiaccounts/settings/database_test.go +++ b/multiaccounts/settings/database_test.go @@ -320,3 +320,49 @@ func TestDatabase_BackupPath(t *testing.T) { require.NoError(t, err) require.Equal(t, testPath, settings.BackupPath) } + +func TestDatabase_ThirdpartyServicesEnabled(t *testing.T) { + db, stop := setupTestDB(t) + defer stop() + + require.NoError(t, db.CreateSettings(settings, config)) + + // By default, third-party services should be enabled + enabled, err := db.ThirdpartyServicesEnabled() + require.NoError(t, err) + require.True(t, enabled, "expected ThirdpartyServicesEnabled to be true by default") + + // Disable third-party services + err = db.SaveSetting(ThirdpartyServicesEnabled.GetReactName(), false) + require.NoError(t, err) + + settings, err = db.GetSettings() + require.NoError(t, err) + require.False(t, settings.ThirdpartyServicesEnabled, "expected ThirdpartyServicesEnabled to be false after disabling") + + // Re-enable third-party services + err = db.SaveSetting(ThirdpartyServicesEnabled.GetReactName(), true) + require.NoError(t, err) + + settings, err = db.GetSettings() + require.NoError(t, err) + require.True(t, settings.ThirdpartyServicesEnabled, "expected ThirdpartyServicesEnabled to be true after enabling") +} + +func TestDatabase_MessagesBackupEnabled(t *testing.T) { + db, stop := setupTestDB(t) + defer stop() + + require.NoError(t, db.CreateSettings(settings, config)) + + enabled, err := db.MessagesBackupEnabled() + require.NoError(t, err) + require.Equal(t, false, enabled) + + err = db.SaveSetting(MessagesBackupEnabled.GetReactName(), true) + require.NoError(t, err) + + settings, err = db.GetSettings() + require.NoError(t, err) + require.Equal(t, true, settings.MessagesBackupEnabled) +} diff --git a/multiaccounts/settings/structs.go b/multiaccounts/settings/structs.go index 0472c5e1fc6..d41e296f4ec 100644 --- a/multiaccounts/settings/structs.go +++ b/multiaccounts/settings/structs.go @@ -215,6 +215,7 @@ type Settings struct { TelemetryServerURL string `json:"telemetry-server-url,omitempty"` TelemetrySendPeriodMs int `json:"telemetry-send-period-ms,omitempty"` BackupPath string `json:"backup-path,omitempty"` + MessagesBackupEnabled bool `json:"messages-backup-enabled?,omitempty"` AutoMessageEnabled bool `json:"auto-message-enabled?,omitempty"` GifAPIKey string `json:"gifs/api-key"` TestNetworksEnabled bool `json:"test-networks-enabled?,omitempty"` diff --git a/protocol/common/message_test.go b/protocol/common/message_test.go index e8b779e1552..40d301d2a4a 100644 --- a/protocol/common/message_test.go +++ b/protocol/common/message_test.go @@ -3,7 +3,7 @@ package common import ( "encoding/base64" "encoding/json" - "io/ioutil" + "io" "os" "testing" @@ -22,7 +22,7 @@ func TestPrepareContentImage(t *testing.T) { require.NoError(t, err) defer file.Close() - payload, err := ioutil.ReadAll(file) + payload, err := io.ReadAll(file) require.NoError(t, err) message := NewMessage() @@ -42,7 +42,7 @@ func TestPrepareContentAudio(t *testing.T) { require.NoError(t, err) defer file.Close() - payload, err := ioutil.ReadAll(file) + payload, err := io.ReadAll(file) require.NoError(t, err) message := NewMessage() diff --git a/protocol/message_persistence.go b/protocol/message_persistence.go index 017ded46834..1fa4677daf4 100644 --- a/protocol/message_persistence.go +++ b/protocol/message_persistence.go @@ -12,6 +12,8 @@ import ( "github.com/golang/protobuf/proto" "github.com/lib/pq" + "github.com/status-im/markdown" + "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/protobuf" ) @@ -122,6 +124,37 @@ func (db sqlitePersistence) tableUserMessagesAllFields() string { payment_requests` } +func (db sqlitePersistence) tableUserMessagesProtobufFields() string { + return ` + m1.id, + m1.whisper_timestamp, + m1.text, + m1.source, + m1.response_to, + m1.chat_id, + m1.message_type, + m1.content_type, + + m1.sticker_pack, + m1.sticker_hash, + + m1.image_payload, + m1.image_type, + + COALESCE(m1.album_id, ""), + COALESCE(m1.album_images_count, 0), + COALESCE(m1.image_width, 0), + COALESCE(m1.image_height, 0), + + m1.links, + m1.unfurled_links, + m1.unfurled_status_links, + + m1.payment_requests, + + pm.pinned_by` +} + // keep the same order as in tableUserMessagesScanAllFields func (db sqlitePersistence) tableUserMessagesAllFieldsJoin() string { return `m1.id, @@ -526,6 +559,91 @@ func (db sqlitePersistence) tableUserMessagesScanAllFields(row scanner, message return nil } +// keep the same order as in tableUserMessagesProtobufFields +func (db sqlitePersistence) tableUserMessagesScanProtobufFields(row scanner, message *protobuf.BackedUpMessage, others ...interface{}) error { + + sticker := &protobuf.StickerMessage{} + image := &protobuf.ImageMessage{} + var serializedLinks []byte + var serializedUnfurledLinks []byte + var serializedPaymentRequests []byte + var serializedUnfurledStatusLinks []byte + var pinnedBy sql.NullString + + args := []interface{}{ + &message.Id, + &message.Timestamp, + &message.Text, + &message.From, // source in table + &message.ResponseTo, + &message.ChatId, + &message.MessageType, + &message.ContentType, + + &sticker.Pack, + &sticker.Hash, + + &image.Payload, + &image.Format, + + &image.AlbumId, + &image.AlbumImagesCount, + &image.Width, + &image.Height, + + &serializedLinks, + &serializedUnfurledLinks, + &serializedUnfurledStatusLinks, + + &serializedPaymentRequests, + + &pinnedBy, + } + err := row.Scan(append(args, others...)...) + if err != nil { + return err + } + + if serializedUnfurledLinks != nil { + err = json.Unmarshal(serializedUnfurledLinks, &message.UnfurledLinks) + if err != nil { + return err + } + } + + if serializedUnfurledStatusLinks != nil { + // use proto.Marshal, because json.Marshal doesn't support `oneof` fields + var links protobuf.UnfurledStatusLinks + err = proto.Unmarshal(serializedUnfurledStatusLinks, &links) + if err != nil { + return err + } + message.UnfurledStatusLinks = &links + } + + if serializedPaymentRequests != nil { + err := json.Unmarshal(serializedPaymentRequests, &message.PaymentRequests) + if err != nil { + return err + } + } + + if pinnedBy.Valid { + message.PinnedBy = pinnedBy.String + } + + switch message.ContentType { + case int64(protobuf.ChatMessage_STICKER): + message.Payload = &protobuf.BackedUpMessage_Sticker{Sticker: sticker} + + case int64(protobuf.ChatMessage_IMAGE): + message.Payload = &protobuf.BackedUpMessage_Image{Image: image} + + } + + return nil +} + func (db sqlitePersistence) tableUserMessagesAllValues(message *common.Message) ([]interface{}, error) { var gapFrom, gapTo uint32 @@ -1335,6 +1453,229 @@ func (db sqlitePersistence) MessageByChatIDs(chatIDs []string, currCursor string return result, newCursor, nil } +func (db sqlitePersistence) AllMessagesForBackup() ([]*protobuf.BackedUpMessage, error) { + where := "WHERE NOT(m1.hide)" + fields := db.tableUserMessagesProtobufFields() + selectQuery := `SELECT %s + FROM user_messages m1 + LEFT JOIN pin_messages pm + ON m1.id = pm.message_id AND pm.pinned = 1` + query := fmt.Sprintf(selectQuery, fields) + " " + where + rows, err := db.db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var messages []*protobuf.BackedUpMessage + for rows.Next() { + message := &protobuf.BackedUpMessage{} + if err := db.tableUserMessagesScanProtobufFields(rows, message); err != nil { + return nil, err + } + + messages = append(messages, message) + } + + return messages, nil +} + +func (db sqlitePersistence) saveBackedUpMessages(messages []*protobuf.BackedUpMessage) error { + if len(messages) == 0 { + return nil + } + + tx, err := db.db.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return err + } + defer func() { + if err == nil { + err = tx.Commit() + return + } + // don't shadow original error + _ = tx.Rollback() + }() + + allFields := db.tableUserMessagesAllFields() + valuesVector := strings.Repeat("?, ", db.tableUserMessagesAllFieldsCount()-1) + "?" + query := "INSERT OR REPLACE INTO user_messages(" + allFields + ") VALUES (" + valuesVector + ")" //nolint: gosec + stmt, err := tx.Prepare(query) + if err != nil { + return err + } + defer stmt.Close() + + for _, msg := range messages { + allValues, err := db.backedUpMessageToUserMessageValues(msg) + if err != nil { + return err + } + + _, err = stmt.Exec(allValues...) + if err != nil { + return err + } + } + + return nil +} + +func (db sqlitePersistence) saveBackedUpPinMessages(messages []*protobuf.BackedUpMessage) error { + pinMessages := make([]*common.PinMessage, 0) + + for _, msg := range messages { + if msg.PinnedBy == "" { + continue + } + + pinMessage := protobuf.PinMessage{ + Clock: msg.Timestamp, + MessageId: msg.Id, + ChatId: msg.ChatId, + MessageType: msg.MessageType, + Pinned: true, + } + pinMessages = append(pinMessages, &common.PinMessage{ + ID: msg.Id, // TODO do we need a special ID? + From: msg.PinnedBy, + PinMessage: &pinMessage, + WhisperTimestamp: msg.Timestamp, + }) + } + + if len(pinMessages) == 0 { + return nil + } + + return db.SavePinMessages(pinMessages) +} + +func (db sqlitePersistence) SaveBackedUpMessages(messages []*protobuf.BackedUpMessage) error { + if len(messages) == 0 { + return nil + } + + err := db.saveBackedUpMessages(messages) + if err != nil { + return err + } + + err = db.saveBackedUpPinMessages(messages) + if err != nil { + return err + } + + return nil +} + +func (db sqlitePersistence) backedUpMessageToUserMessageValues(message *protobuf.BackedUpMessage) ([]interface{}, error) { + parsedText := markdown.Parse([]byte(message.Text), nil) + jsonParsedText, err := json.Marshal(parsedText) + if err != nil { + return nil, err + } + + sticker := message.GetSticker() + if sticker == nil { + sticker = &protobuf.StickerMessage{} + } + + image := message.GetImage() + if image == nil { + image = &protobuf.ImageMessage{} + } + + var serializedUnfurledLinks []byte + if links := message.GetUnfurledLinks(); len(links) > 0 { + serializedUnfurledLinks, err = json.Marshal(links) + if err != nil { + return nil, err + } + } + + var serializedUnfurledStatusLinks []byte + if links := message.GetUnfurledStatusLinks(); links != nil { + // use proto.Marshal, because json.Marshal doesn't support `oneof` fields + serializedUnfurledStatusLinks, err = proto.Marshal(links) + if err != nil { + return nil, err + } + } + + var serializedPaymentRequests []byte + if paymentRequests := message.GetPaymentRequests(); len(paymentRequests) > 0 { + serializedPaymentRequests, err = json.Marshal(paymentRequests) + if err != nil { + return nil, err + } + } + + // Convert protobuf MessageType to the expected format + messageType := message.MessageType + + return []interface{}{ + message.GetId(), // id + message.GetTimestamp(), // whisper_timestamp + message.GetFrom(), // source + message.GetText(), // text + message.GetContentType(), // content_type + "", // username (alias) - not available in BackedUpMessage + message.GetTimestamp(), // timestamp + message.GetChatId(), // chat_id + message.GetChatId(), // local_chat_id (same as chat_id) // TODO this should be adapated if 1-1 + messageType, // message_type + message.GetClock(), // clock_value + false, // seen + "", // outgoing_status + jsonParsedText, // parsed_text + sticker.GetPack(), // sticker_pack + sticker.GetHash(), // sticker_hash + image.GetPayload(), // image_payload + int64(image.GetFormat()), // image_type + image.GetAlbumId(), // album_id + nil, // album_images + image.GetAlbumImagesCount(), // album_images_count + image.GetWidth(), // image_width + image.GetHeight(), // image_height + "", // image_base64 + nil, // audio_payload + 0, // audio_type + 0, // audio_duration_ms + "", // audio_base64 + "", // community_id + nil, // mentions + nil, // links + serializedUnfurledLinks, // unfurled_links + serializedUnfurledStatusLinks, // unfurled_status_links + "", // command_id + "", // command_value + "", // command_from + "", // command_address + "", // command_contract + "", // command_transaction_hash + 0, // command_state + nil, // command_signature + "", // replace + int64(0), // edited_at + false, // deleted + "", // deleted_by + false, // deleted_for_me + false, // rtl + 0, // line_count + message.GetResponseTo(), // response_to + uint32(0), // gap_from + uint32(0), // gap_to + 0, // contact_request_state + 0, // contact_verification_state + false, // mentioned + false, // replied + "", // discord_message_id + serializedPaymentRequests, // payment_requests + }, nil +} + func (db sqlitePersistence) OldestMessageWhisperTimestampByChatIDs(chatIDs []string) (map[string]uint64, error) { if len(chatIDs) == 0 { return nil, nil diff --git a/protocol/messenger_backup_handler.go b/protocol/messenger_backup_handler.go index 60e0ccf62a3..e48d9bc8c85 100644 --- a/protocol/messenger_backup_handler.go +++ b/protocol/messenger_backup_handler.go @@ -27,31 +27,38 @@ const ( SyncWakuSectionKeyWatchOnlyAccounts = "watchOnlyAccounts" ) -func (m *Messenger) handleLocalBackup(state *ReceivedMessageState, message *protobuf.MessengerLocalBackup) []error { +func (m *Messenger) handleLocalBackup(state *ReceivedMessageState, backup *protobuf.MessengerLocalBackup) []error { var errors []error - err := m.handleBackedUpProfile(message.Profile, message.Clock) + err := m.handleBackedUpProfile(backup.Profile, backup.Clock) if err != nil { errors = append(errors, err) } - for _, contact := range message.Contacts { + for _, contact := range backup.Contacts { err = m.HandleSyncInstallationContactV2(state, contact, nil) if err != nil { errors = append(errors, err) } } - err = m.handleSyncChats(state, message.Chats) + err = m.handleSyncChats(state, backup.Chats) if err != nil { errors = append(errors, err) } - communityErrors := m.handleLocalBackupCommunities(state, message.Communities) + communityErrors := m.handleLocalBackupCommunities(state, backup.Communities) if len(communityErrors) > 0 { errors = append(errors, communityErrors...) } + if len(backup.Messages) > 0 { + err := m.persistence.SaveBackedUpMessages(backup.Messages) + if err != nil { + errors = append(errors, err) + } + } + return errors } @@ -231,7 +238,7 @@ func (m *Messenger) handleLocalBackupCommunities(state *ReceivedMessageState, co errors = append(errors, err) } - err = m.requestCommunityKeysAndSharedAddresses(state, syncCommunity) + err = m.requestCommunityKeysAndSharedAddresses(syncCommunity) if err != nil { errors = append(errors, err) } @@ -240,7 +247,7 @@ func (m *Messenger) handleLocalBackupCommunities(state *ReceivedMessageState, co return errors } -func (m *Messenger) requestCommunityKeysAndSharedAddresses(state *ReceivedMessageState, syncCommunity *protobuf.SyncInstallationCommunity) error { +func (m *Messenger) requestCommunityKeysAndSharedAddresses(syncCommunity *protobuf.SyncInstallationCommunity) error { if !syncCommunity.Joined { return nil } diff --git a/protocol/messenger_local_backup.go b/protocol/messenger_local_backup.go index a5ece9e685e..3fac610c3ce 100644 --- a/protocol/messenger_local_backup.go +++ b/protocol/messenger_local_backup.go @@ -233,6 +233,19 @@ func (m *Messenger) backupProfile(clock uint64) (*protobuf.Backup, error) { return backupMessage, nil } +func (m *Messenger) backupMessages() ([]*protobuf.BackedUpMessage, error) { + messagesBackupEnabled, err := m.settings.MessagesBackupEnabled() + if err != nil { + return nil, err + } + + if !messagesBackupEnabled { + return nil, nil + } + + return m.persistence.AllMessagesForBackup() +} + func (m *Messenger) ExportBackup() ([]byte, error) { backup := &protobuf.MessengerLocalBackup{} @@ -258,6 +271,13 @@ func (m *Messenger) ExportBackup() ([]byte, error) { for _, d := range chatsToBackup { backup.Chats = append(backup.Chats, d.Chats...) } + + backupMessages, err := m.backupMessages() + if err != nil { + return nil, err + } + backup.Messages = backupMessages + return proto.Marshal(backup) } diff --git a/protocol/messenger_local_backup_test.go b/protocol/messenger_local_backup_test.go index 3c565e85597..9c0c77d5e0e 100644 --- a/protocol/messenger_local_backup_test.go +++ b/protocol/messenger_local_backup_test.go @@ -2,12 +2,16 @@ package protocol import ( "context" + "io" + "os" "testing" "github.com/stretchr/testify/suite" - "github.com/status-im/status-go/eth-node/crypto" - "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/crypto" + "github.com/status-im/status-go/crypto/types" + "github.com/status-im/status-go/multiaccounts/settings" + "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/requests" ) @@ -29,6 +33,15 @@ func (s *MessengerLocalBackupSuite) TestLocalBackup() { bob2 := s.anotherMessenger() defer TearDownMessenger(&s.Suite, bob2) + // Enable message backup on both accounts + err := bob1.settings.SaveSetting(settings.MessagesBackupEnabled.GetReactName(), true) + s.Require().NoError(err) + + err = bob2.settings.SaveSetting(settings.MessagesBackupEnabled.GetReactName(), true) + s.Require().NoError(err) + + ctx := context.Background() + // -------------------- CONTACTS -------------------- // Create 2 contacts contact1Key, err := crypto.GenerateKey() @@ -77,6 +90,81 @@ func (s *MessengerLocalBackupSuite) TestLocalBackup() { s.Require().NotNil(response) s.Require().Len(response.Communities(), 1) + // Send message on community + chatID := response.Chats()[0].ID + inputMessage := common.NewMessage() + inputMessage.ChatId = chatID + inputMessage.ContentType = protobuf.ChatMessage_TEXT_PLAIN + inputMessage.Text = "some text" + + _, err = bob1.SendChatMessage(ctx, inputMessage) + s.Require().NoError(err) + + // Pin message + pinMessage := common.NewPinMessage() + pinMessage.ChatId = chatID + pinMessage.MessageId = inputMessage.ID + pinMessage.Pinned = true + sendResponse, err := bob1.SendPinMessage(ctx, pinMessage) + s.Require().NoError(err) + s.Require().Len(sendResponse.PinMessages(), 1) + + // Send markdown message + mdMessage := common.NewMessage() + mdMessage.ChatId = chatID + mdMessage.ContentType = protobuf.ChatMessage_TEXT_PLAIN + mdMessage.Text = "some *markdown* text" + + _, err = bob1.SendChatMessage(ctx, mdMessage) + s.Require().NoError(err) + + // Send image on community + file, err := os.Open("../_assets/tests/test.jpg") + s.Require().NoError(err) + defer file.Close() + + payload, err := io.ReadAll(file) + s.Require().NoError(err) + + imageMessage := common.NewMessage() + imageMessage.ChatId = chatID + imageMessage.ContentType = protobuf.ChatMessage_IMAGE + + image := protobuf.ImageMessage{ + Payload: payload, + Format: protobuf.ImageFormat_JPEG, + Width: 1200, + Height: 1000, + AlbumId: "", + } + imageMessage.Payload = &protobuf.ChatMessage_Image{Image: &image} + imageMessage.Text = "some image" + + _, err = bob1.SendChatMessage(ctx, imageMessage) + s.Require().NoError(err) + + // Send sticker on community + stickerMessage := common.NewMessage() + stickerMessage.ChatId = chatID + stickerMessage.ContentType = protobuf.ChatMessage_STICKER + stickerMessage.Text = "some sticker" + stickerMessage.Payload = &protobuf.ChatMessage_Sticker{ + Sticker: &protobuf.StickerMessage{ + Pack: 1, + Hash: "some-hash", + }, + } + _, err = bob1.SendChatMessage(ctx, stickerMessage) + s.Require().NoError(err) + + // Send emoji on community + emojiMessage := common.NewMessage() + emojiMessage.ChatId = chatID + emojiMessage.ContentType = protobuf.ChatMessage_EMOJI + emojiMessage.Text = ":+1:" + _, err = bob1.SendChatMessage(ctx, emojiMessage) + s.Require().NoError(err) + // Check bob2 communities, err := bob2.Communities() s.Require().NoError(err) @@ -127,6 +215,14 @@ func (s *MessengerLocalBackupSuite) TestLocalBackup() { err = bob1.SaveChat(ourOneOneChat) s.Require().NoError(err) + // Send transaction command to Alice + transactionMessage := common.NewMessage() + transactionMessage.ChatId = ourOneOneChat.ID + transactionMessage.ContentType = protobuf.ChatMessage_TRANSACTION_COMMAND + transactionMessage.Text = "some transaction" + _, err = bob1.SendChatMessage(ctx, transactionMessage) + s.Require().NoError(err) + // -------------------- BACKUP -------------------- // Backup marshalledBackup, err := bob1.ExportBackup() @@ -156,4 +252,50 @@ func (s *MessengerLocalBackupSuite) TestLocalBackup() { s.Require().True(ok) s.Require().True(chat.Active) s.Require().Equal("", chat.Name) + + // Validate messages + s.Require().NoError(err) + messages, err := bob2.persistence.AllMessagesForBackup() + s.Require().NoError(err) + s.Require().Len(messages, 14) + + textMessageFound := false + mdMessageFound := false + imageMessageFound := false + stickerMessageFound := false + emojiMessageFound := false + txMessageFound := false + pinnedSystemMessageFound := false + for _, msg := range messages { + if msg.ContentType == int64(protobuf.ChatMessage_TEXT_PLAIN) && msg.Text == "some text" { + textMessageFound = true + s.Require().True(msg.PinnedBy == bob2.selfContact.ID) + } else if msg.ContentType == int64(protobuf.ChatMessage_TEXT_PLAIN) && msg.Text == "some *markdown* text" { + mdMessageFound = true + } else if msg.ContentType == int64(protobuf.ChatMessage_IMAGE) && msg.Text == "some image" { + imageMessageFound = true + } else if msg.ContentType == int64(protobuf.ChatMessage_STICKER) && msg.Text == "some sticker" { + stickerMessageFound = true + } else if msg.ContentType == int64(protobuf.ChatMessage_EMOJI) && msg.Text == ":+1:" { + emojiMessageFound = true + } else if msg.ContentType == int64(protobuf.ChatMessage_TRANSACTION_COMMAND) && msg.Text == "some transaction" { + txMessageFound = true + } else if msg.ContentType == int64(protobuf.ChatMessage_SYSTEM_MESSAGE_PINNED_MESSAGE) && msg.Text == "" { + pinnedSystemMessageFound = true + } + } + s.Require().True(textMessageFound) + s.Require().True(mdMessageFound) + s.Require().True(imageMessageFound) + s.Require().True(stickerMessageFound) + s.Require().True(emojiMessageFound) + s.Require().True(txMessageFound) + s.Require().True(pinnedSystemMessageFound) + + // Validate pinned messages + pinnedMessages, _, err := bob2.PinnedMessageByChatID(chatID, "", 10) + s.Require().NoError(err) + s.Require().Len(pinnedMessages, 1) + s.Require().Equal(bob2.selfContact.ID, pinnedMessages[0].PinnedBy) + } diff --git a/protocol/messenger_test.go b/protocol/messenger_test.go index de806b99843..11e438ba348 100644 --- a/protocol/messenger_test.go +++ b/protocol/messenger_test.go @@ -2416,13 +2416,13 @@ func (s *MessengerSuite) TestResendExpiredEmojis() { func buildImageWithAlbumIDMessage(chat Chat, albumID string) (*common.Message, error) { file, err := os.Open("../_assets/tests/test.jpg") - if err != err { + if err != nil { return nil, err } defer file.Close() payload, err := io.ReadAll(file) - if err != err { + if err != nil { return nil, err } diff --git a/protocol/protobuf/messenger_local_backup.proto b/protocol/protobuf/messenger_local_backup.proto index a8975a3e5e1..226a5d11eff 100644 --- a/protocol/protobuf/messenger_local_backup.proto +++ b/protocol/protobuf/messenger_local_backup.proto @@ -1,6 +1,8 @@ syntax = "proto3"; import "pairing.proto"; +import "chat_message.proto"; +import "enums.proto"; option go_package = "./;protobuf"; package protobuf; @@ -13,4 +15,47 @@ message MessengerLocalBackup { repeated SyncInstallationCommunity communities = 3; repeated SyncChat chats = 4; BackedUpProfile profile = 5; -} \ No newline at end of file + repeated BackedUpMessage messages = 6; +} + +message BackedUpMessage { + // Lamport timestamp of the chat message + uint64 clock = 1; + + // ID generated from the sender's public key and the message payload + string id = 2; + // The public key of the sender of the message + string from = 3; + + // Unix timestamps in milliseconds, currently not used as we use whisper as + // more reliable, but here so that we don't rely on it + uint64 timestamp = 4; + // Text of the message + string text = 5; + // Id of the message that we are replying to + string response_to = 6; + // Chat id, this field is symmetric for public-chats and private group chats, + // but asymmetric in case of one-to-ones, as the sender will use the chat-id + // of the received, while the receiver will use the chat-id of the sender. + // Probably should be the concatenation of sender-pk & receiver-pk in + // alphabetical order + string chat_id = 7; + + // The type of message (public/one-to-one/private-group-chat) + MessageType message_type = 8; + // The type of the content of the message + int64 content_type = 9; + + oneof payload { + StickerMessage sticker = 10; + ImageMessage image = 11; + } + + repeated UnfurledLink unfurled_links = 12; + + UnfurledStatusLinks unfurled_status_links = 13; + + repeated PaymentRequest payment_requests = 14; + + string pinned_by = 15; +} diff --git a/services/accounts/settings.go b/services/accounts/settings.go index 075dfafc5c7..c6246568bae 100644 --- a/services/accounts/settings.go +++ b/services/accounts/settings.go @@ -68,6 +68,10 @@ func (api *SettingsAPI) BackupPath() (string, error) { return api.db.BackupPath() } +func (api *SettingsAPI) MessagesBackupEnabled() (bool, error) { + return api.db.MessagesBackupEnabled() +} + // Notifications Settings func (api *SettingsAPI) NotificationsGetAllowNotifications() (bool, error) { return api.db.GetAllowNotifications() From 16d965ed769bebbd91f04403e03f4aa1407eeba0 Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Thu, 11 Sep 2025 15:04:57 -0400 Subject: [PATCH 14/15] fix(settings): fix wrongly named reactFieldName for messages backup --- multiaccounts/settings/columns.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiaccounts/settings/columns.go b/multiaccounts/settings/columns.go index 7ec3ca2eedb..d3775192447 100644 --- a/multiaccounts/settings/columns.go +++ b/multiaccounts/settings/columns.go @@ -172,7 +172,7 @@ var ( dBColumnName: "log_level", } MessagesBackupEnabled = SettingField{ - reactFieldName: "messages-backup-enabled", + reactFieldName: "messages-backup-enabled?", dBColumnName: "messages_backup_enabled", valueHandler: BoolHandler, } From cd0f04de2a1f22426d2c8bf9319da0c92995f373 Mon Sep 17 00:00:00 2001 From: Jonathan Rainville Date: Tue, 16 Sep 2025 14:04:45 -0400 Subject: [PATCH 15/15] feat(backup): send a signal to reset chat cursors on message imports Needed for https://github.com/status-im/status-desktop/issues/18824 --- protocol/messenger_backup_handler.go | 3 +++ signal/events_shhext.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/protocol/messenger_backup_handler.go b/protocol/messenger_backup_handler.go index e48d9bc8c85..bd159a0b5e9 100644 --- a/protocol/messenger_backup_handler.go +++ b/protocol/messenger_backup_handler.go @@ -16,6 +16,7 @@ import ( "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/wakusync" ensservice "github.com/status-im/status-go/services/ens" + "github.com/status-im/status-go/signal" ) const ( @@ -57,6 +58,8 @@ func (m *Messenger) handleLocalBackup(state *ReceivedMessageState, backup *proto if err != nil { errors = append(errors, err) } + + signal.LocalMessageBackupDone() } return errors diff --git a/signal/events_shhext.go b/signal/events_shhext.go index 390cb6a7574..7603fb73b5c 100644 --- a/signal/events_shhext.go +++ b/signal/events_shhext.go @@ -39,6 +39,9 @@ const ( // EventNewMessages is triggered when we receive new messages EventNewMessages = "messages.new" + // EventLocalMessageBackupDone is triggered when a local message backup is completed + EventLocalMessageBackupDone = "local.message.backup.done" + // EventHistoryRequestStarted is triggered before processing a store request EventHistoryRequestStarted = "history.request.started" @@ -213,6 +216,10 @@ func SendNewMessages(obj json.Marshaler) { send(EventNewMessages, obj) } +func LocalMessageBackupDone() { + send(EventLocalMessageBackupDone, interface{}(nil)) +} + func sendMailserverSignal(ms *wakutypes.Mailserver, event string) { msSignal := MailserverSignal{} if ms != nil {