Merge branch 'master' into feature/go-native-trace

Author: palango

.circleci/config.yml

@@ -262,6 +262,7 @@ end-to-end-test: &end-to-end-test
         no_output_timeout: 15m
         command: |
           export PATH=${PATH}:~/repos/golang/go/bin
+          export E2E_TESTS_FORCE_USE_MYCELO=true
           cd celo-monorepo/packages/celotool
           ./${TEST_NAME} local ~/repos/geth
     # Note, all e2e tests call 'make all' in ~/repos/geth, this causes most code
@@ -278,6 +279,34 @@ jobs:
       - run: go mod download
       - *save-go-mod-cache
 
+  akeyless-get-token:
+    docker:
+      - image: akeyless/ci_base
+    environment:
+      PR_WRITE_TOKEN_PATH: /dynamic-secrets/keys/github/celo-blockchain/pull_requests=write
+    steps:
+      - checkout
+      - run:
+          name: "update akeyless cli tool to latest version"
+          command: akeyless update
+
+      - run:
+          name: "authenticate to akeyless via OIDC"
+          command: akeyless auth --access-id $accessid --access-type jwt --jwt $CIRCLE_OIDC_TOKEN --json | jq '.token' -r  > ~/.vault_token
+
+      - run:
+          name: "Get PR Comment token (and remove unnecessary information and punctuation)"
+          command: TOKEN=$(cat ~/.vault_token) && echo export PR_COMMENT_TOKEN2=$(akeyless get-dynamic-secret-value --name $PR_WRITE_TOKEN_PATH --token $TOKEN |grep token | awk '{print $2}' | tr -d '"",' ) >> "$BASH_ENV"
+
+      - run:
+          name: "Copy $BASH_ENV to bash.env file so we can persist that to workspaces"
+          command: cp $BASH_ENV bash.env
+
+      - persist_to_workspace:
+          root: .
+          paths:
+            - bash.env
+
   prepare-system-contracts:
     parameters:
       cache-key:
@@ -376,6 +405,11 @@ jobs:
             go run tools/parsecov/main.go -packagePrefix github.com/celo-org/celo-blockchain/ cov.out > summary
             cat summary
 
+      - attach_workspace:
+          at: .
+      - run: |
+          cat bash.env > $BASH_ENV
+
       - run:
           name: Post summary comment on PR
           command: |
@@ -435,7 +469,8 @@ jobs:
             # replaced by '\n'. Using backtics causes there to be a round of
             # backslash processing on the command before execution, so we
             # need to double the backslashes in the awk command.
-            curl -u piersy:${PR_COMMENT_TOKEN} -X ${CURL_VERB} $URL -d "{\"body\":\"`awk -v ORS='\\\\n' '1' comment`\"}" ;
+            curl -u celo-org:${PR_COMMENT_TOKEN2} -X ${CURL_VERB} $URL -d "{\"body\":\"`awk -v ORS='\\\\n' '1' comment`\"}" ;
+
 
   e2e-benchmarks:
     executor: golang
@@ -499,40 +534,31 @@ jobs:
           # paths:
           #  - geth/build/bin/geth.aar
 
-  ios:
-    macos:
-      xcode: "12.5.1"
-    working_directory: ~/repos/geth
-    steps:
-      # Note the macos executor does not seem to be able to restore caches.
-      - *shallow-checkout
-      - run:
-          name: Setup Go language
-          command: |
-            brew install [email protected]
-            brew link [email protected]
-            # Check that homebrew installed the expected go version
-            if [[ "$(go version)" != "go version go1.16"* ]]; then
-              echo "go1.16 is required"
-              exit 1
-            fi
-      - run:
-          name: Compile ios client
-          command: make ios
-      - persist_to_workspace:
-          root: ~/repos
-          paths:
-            - geth/build/bin/Geth.framework.tgz
-            - geth/libbls_snark_sys.a
-
-  publish-mobile-client:
-    docker:
-      - image: circleci/node:10
-    working_directory: ~/repos/geth
-    steps:
-      - attach_workspace:
-          at: ~/repos
-      - run: ./scripts/publish-mobile-client.sh ${CIRCLE_SHA1} ${NPM_TOKEN_FOR_CELO_CLIENT}
+  # ios:
+  #   macos:
+  #     xcode: "12.5.1"
+  #   working_directory: ~/repos/geth
+  #   steps:
+  #     # Note the macos executor does not seem to be able to restore caches.
+  #     - *shallow-checkout
+  #     - run:
+  #         name: Setup Go language
+  #         command: |
+  #           brew install [email protected]
+  #           brew link [email protected]
+  #           # Check that homebrew installed the expected go version
+  #           if [[ "$(go version)" != "go version go1.16"* ]]; then
+  #             echo "go1.16 is required"
+  #             exit 1
+  #           fi
+  #     - run:
+  #         name: Compile ios client
+  #         command: make ios
+  #     - persist_to_workspace:
+  #         root: ~/repos
+  #         paths:
+  #           - geth/build/bin/Geth.framework.tgz
+  #           - geth/libbls_snark_sys.a
 
   lightest-sync-test:
     executor: golang
@@ -623,34 +649,20 @@ workflows:
           requires:
             - go-modules
             - prepare-system-contracts
+      - akeyless-get-token:
+          context:
+            - akeyless
       - istanbul-e2e-coverage:
           requires:
             - go-modules
             - prepare-system-contracts
+            - akeyless-get-token
       - e2e-benchmarks:
           requires:
             - go-modules
             - prepare-system-contracts
-              #- android
-      - ios
-      - publish-mobile-client:
-          requires:
-            #- android
-            - ios
-            # Makes sure tests are all green before publishing
-            # Though these are not using the mobile built binaries
-            # they should be a good indicator
-            - unit-tests
-            - lightest-sync-test
-            - end-to-end-blockchain-parameters-test
-            - end-to-end-governance-test
-            - end-to-end-slashing-test
-            - end-to-end-sync-test
-            - end-to-end-transfer-test
-            - end-to-end-validator-order-test
-          filters:
-            branches:
-              only: master
+      # - android
+      # - ios
       - lightest-sync-test:
           requires:
             - go-modules

.github/ISSUE_TEMPLATE/chore.md

@@ -2,7 +2,7 @@
 name: Simple Task
 about: Simple Actionable Task
 title: ''
-labels: blockchain
+labels: triage
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/design_task.md

@@ -2,7 +2,7 @@
 name: Research/Design Task
 about: Research of design task
 title: ''
-labels: blockchain, needs-research
+labels: triage, needs-research
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/feature.md

@@ -2,7 +2,7 @@
 name: Request a feature
 about: Report a missing feature - e.g. as a step before submitting a PR
 title: ''
-labels: 'type:feature'
+labels: triage, 'type:feature'
 assignees: ''
 ---
 
@@ -14,4 +14,4 @@ What are the use-cases?
 # Implementation
 
 Do you have ideas regarding the implementation of this feature?
-Are you willing to implement this feature?
+Are you willing to implement this feature?

.github/ISSUE_TEMPLATE/issue.md

@@ -2,7 +2,7 @@
 name: Issue
 about: Feature/Need/Problem complete template
 title: ''
-labels: blockchain, needs-research
+labels: triage, needs-research
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/question.md

@@ -2,7 +2,7 @@
 name: Ask a question
 about: Something is unclear
 title: ''
-labels: 'type:docs'
+labels: triage, 'type:docs'
 assignees: ''
 ---
 

CeloBlockchain.podspec

@@ -6,7 +6,7 @@ Official golang implementation of the Celo blockchain, based off of the [officia
 [![Codecov](https://img.shields.io/codecov/c/github/celo-org/celo-blockchain)](https://codecov.io/gh/celo-org/celo-blockchain)
 [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://chat.celo.org)
 
-Prebuilt [Docker](https://en.wikipedia.org/wiki/Docker_\(software\)) images are available for immediate use: [us.gcr.io/celo-testnet/celo-node](https://us.gcr.io/celo-testnet/celo-node). See [docs.celo.org/getting-started](https://docs.celo.org/getting-started/choosing-a-network) for a guide to the Celo networks and how to get started.
+Prebuilt [Docker](https://en.wikipedia.org/wiki/Docker_\(software\)) images are available for immediate use: [us.gcr.io/celo-org/geth](https://us.gcr.io/celo-org/geth). See [docs.celo.org/getting-started](https://docs.celo.org/getting-started/choosing-a-network) for a guide to the Celo networks and how to get started.
 
 Documentation for Celo more generally can be found at [docs.celo.org](https://docs.celo.org/)
 

README.md

@@ -1342,7 +1342,6 @@ func xgoAllToolsArchiveFiles(target string, dir string) []string {
 		executableXgoPath("evm", target, dir),
 		executableXgoPath("geth", target, dir),
 		executableXgoPath("rlpdump", target, dir),
-		executableXgoPath("wnode", target, dir),
 		executableXgoPath("clef", target, dir),
 		executableXgoPath("blspopchecker", target, dir),
 	}

build/ci.go

@@ -191,9 +191,10 @@ var (
 		Value: 0,
 	}
 	defaultSyncMode = ethconfig.Defaults.SyncMode
-	SyncModeFlag    = TextMarshalerFlag{
+	// TODO: Check if snap sync is enabled
+	SyncModeFlag = TextMarshalerFlag{
 		Name:  "syncmode",
-		Usage: `Blockchain sync mode ("fast", "full", "snap", "light", or "lightest")`,
+		Usage: `Blockchain sync mode ("fast", "full", "light", or "lightest")`,
 		Value: &defaultSyncMode,
 	}
 	GCModeFlag = cli.StringFlag{
@@ -1732,13 +1733,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100
 	}
 	if !ctx.GlobalBool(SnapshotFlag.Name) {
-		// If snap-sync is requested, this flag is also required
-		if cfg.SyncMode == downloader.SnapSync {
-			log.Info("Snap sync requested, enabling --snapshot")
-		} else {
-			cfg.TrieCleanCache += cfg.SnapshotCache
-			cfg.SnapshotCache = 0 // Disabled
-		}
+		// Snap Dync Disabled. See https://github.com/celo-org/celo-blockchain/issues/1735
+		// // If snap-sync is requested, this flag is also required
+		// if cfg.SyncMode == downloader.SnapSync {
+		// 	log.Info("Snap sync requested, enabling --snapshot")
+		// } else {
+		// 	cfg.TrieCleanCache += cfg.SnapshotCache
+		// 	cfg.SnapshotCache = 0 // Disabled
+		// }
+		cfg.TrieCleanCache += cfg.SnapshotCache
+		cfg.SnapshotCache = 0 // Disabled
 	}
 	if ctx.GlobalIsSet(DocRootFlag.Name) {
 		cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name)

cmd/utils/flags.go

@@ -448,3 +448,29 @@ function call to occur after the given duration.
 // to Hc and value equal to V is greater than 1 and less than 10.
 1 < |{ m<Prepare, Hc, Rm, Vm> ∈ M : Rm = Hc && Vm = V }| < 10
 ```
+
+## Appendix 2: Notes
+
+### What do the prepared certificate and round change certificate do?
+
+They enforce that if it’s possible that a node could have confirmed a block in
+a round, then all remaining nodes in future rounds can only confirm the same
+block.
+
+- A node could have confirmed a block if 2f+1 nodes send commit messages for a
+  block.
+- If a node sends a commit message for a block then it must have seen 2f+1
+  prepares or commits for that block and therefore must have a prepared
+  certificate.
+- Therefore there must be at least 2f+1 nodes with prepared certificates for a
+  block if 2f+1 nodes have sent commit messages for a block.
+- When the proposer proposes in the next round they must provide a round change
+  certificate with 2f+1 round change messages.
+- When nodes send round change messages they should include the prepared
+  certificate if they have one.
+- So if 2f+1 nodes have prepared certificates then f nodes do not.
+- If the proposer of the next round includes the f round change messages from
+  the nodes that do not have prepared certificates, and if there are a further
+  f byzantine nodes that do not include their prepared certificate even if they
+  had one, there still remains one round change message that must come from a
+  non byzantine node and must include a prepared certificate.

consensus/istanbul/celo-ibft-spec.md

@@ -152,10 +152,6 @@ type core struct {
 
 // New creates an Istanbul consensus core
 func New(backend CoreBackend, config *istanbul.Config) Engine {
-	rsdb, err := newRoundStateDB(config.RoundStateDBPath, nil)
-	if err != nil {
-		log.Crit("Failed to open RoundStateDB", "err", err)
-	}
 
 	c := &core{
 		config:                    config,
@@ -167,7 +163,6 @@ func New(backend CoreBackend, config *istanbul.Config) Engine {
 		pendingRequests:           prque.New(nil),
 		pendingRequestsMu:         new(sync.Mutex),
 		consensusTimestamp:        time.Time{},
-		rsdb:                      rsdb,
 		consensusPrepareTimeGauge: metrics.NewRegisteredGauge("consensus/istanbul/core/consensus_prepare", nil),
 		consensusCommitTimeGauge:  metrics.NewRegisteredGauge("consensus/istanbul/core/consensus_commit", nil),
 		verifyGauge:               metrics.NewRegisteredGauge("consensus/istanbul/core/verify", nil),

consensus/istanbul/core/core.go

@@ -22,11 +22,16 @@ import (
 	"github.com/celo-org/celo-blockchain/common"
 	"github.com/celo-org/celo-blockchain/common/hexutil"
 	"github.com/celo-org/celo-blockchain/consensus/istanbul"
+	"github.com/celo-org/celo-blockchain/log"
 )
 
 // Start implements core.Engine.Start
 func (c *core) Start() error {
-
+	rsdb, err := newRoundStateDB(c.config.RoundStateDBPath, nil)
+	if err != nil {
+		log.Crit("Failed to open RoundStateDB", "err", err)
+	}
+	c.rsdb = rsdb
 	roundState, err := c.createRoundState()
 	if err != nil {
 		return err
@@ -63,10 +68,11 @@ func (c *core) Stop() error {
 	// Make sure the handler goroutine exits
 	c.handlerWg.Wait()
 
+	err := c.rsdb.Close()
 	c.currentMu.Lock()
 	defer c.currentMu.Unlock()
 	c.current = nil
-	return nil
+	return err
 }
 
 // ----------------------------------------------------------------------------