Adding proxy command to byzcoin

.github/workflows/test-byzcoin.yaml

@@ -1,8 +1,6 @@
 name: test-byzcoin
 
 on:
-  push:
-    branches: master
   pull_request:
 
 jobs:

.github/workflows/update.yaml

@@ -61,3 +61,59 @@ jobs:
 
       - name: Push new docker image
         run: make docker-push-new
+
+      - name: Store linux binaries
+        uses: actions/upload-artifact@v2
+        with:
+          name: linux-binary
+          path: docker
+
+  release:
+    runs-on: macos-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 1
+
+      - name: Compile Mac binaries
+        run: |
+          MAC=macosx
+          mkdir -p $MAC
+          for bin in byzcoin full; do go build -o $MAC ./cmd/$bin; done
+          ( cd pkg/cothority; \
+            for bin in byzcoin/bcadmin scmgr personhood/phapp; do \
+              go build -o ../../$MAC ./$bin; \
+            done )
+
+      - name: Fetch linux binaries
+        uses: actions/download-artifact@v2
+        with:
+          name: linux-binary
+          path: linux
+
+      - name: show
+        run: ls -R
+
+      - name: Create release.tgz
+        run: |
+          RELEASE_DATE=$(date +%Y-%m-%d)
+          RELEASE=byzcoin-$RELEASE_DATE
+          mkdir $RELEASE
+          ls -l
+          ( cd linux; rm -f Dockerfile built byzcoin.sh )
+          mv linux macosx $RELEASE
+
+          ls -R $RELEASE
+          tar czf byzcoin-linux-macosx-$RELEASE_DATE.tgz $RELEASE
+          echo "RELEASE_DATE=$RELEASE_DATE" >> $GITHUB_ENV
+
+      - name: Create release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "*.tgz"
+          body: "Binary release of the byzcoin files"
+          name: "Release linux and macOSX ${{env.RELEASE_DATE}}"
+          prerelease: true
+          tag: release-${{env.RELEASE_DATE}}
+          token: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -11,9 +11,9 @@ DATE_PREVIOUS := $(shell date --date "last Monday" +%Y%m%d || \
 else
 # mac date doesn't know about --date argument...
 DATE_COMPILE := $(shell date --date "last Monday" +%Y%m%d || \
-               	date -v Mon +%Y%m%d)
+				date -v Mon +%Y%m%d)
 DATE_PREVIOUS := $(shell date --date "Monday a fortnight ago" +%Y%m%d || \
-               	date -v Mon -v -7d +%Y%m%d)
+				date -v Mon -v -7d +%Y%m%d)
 endif
 
 DOCKER_TAG = $(DATE_COMPILE)
@@ -118,6 +118,12 @@ docker-push-all: docker
 		docker push $(DOCKER_NAME):$$d; \
 	done
 
+docker-push-dbg: DATE_COMPILE := $(shell date +%Y%m%d-%H%M)
+docker-push-dbg: NAME := dbg
+docker-push-dbg: docker
+	   docker tag $(DOCKER_NAME):$(DOCKER_TAG) $(DOCKER_NAME):$(NAME)
+	   docker push $(DOCKER_NAME):$(NAME)
+
 verify_latest: LATEST=$(DOCKER_NAME):$(DOCKER_TAG)
 verify_latest: PREVIOUS=$(DOCKER_NAME):$(DATE_PREVIOUS)
 verify_latest: docker

README.md

@@ -269,3 +269,33 @@ This will create a new docker image and tag it for every weekday.
 As the current nodes update their image once an hour, you'll have to wait for
  an hour for every node to update.
 To mark these updates as somewhat not ideal, a `force` is added to the version.
+
+
+# Byzcoin binary
+
+The byzcoin binary has the following commands:
+- `config` to set up a new config
+- `show` to dump a config
+- `run` to start the node as a server
+- `proxy` to start a proxy
+
+The `config`, `show`, and `run` commands are used in the docker and are not
+ more explained here.
+Only the `proxy` command is interesting outside of docker.
+
+## Proxy
+
+You can run a proxy on your local computer for testing, e.g., if you're
+ developing a new web-frontend.
+The proxy takes an existing db, 
+like https://demo.c4dt.org/omniledger/cached.db, and serves it locally.
+When starting, the proxy does the following:
+1. scan all chains in the db and update to the latest block
+2. verify the stored state in the db and eventually updating it
+3. regularly check for updates of the chain
+
+The following arguments can be given to the `proxy` command:
+- `portWS` for the websocket-port where the proxy will listen for commands
+- `update` to change the intervals between two updates
+- `complete` if given, the proxy makes sure all chains have all blocks
+ available 

cmd/byzcoin/main.go

@@ -15,14 +15,20 @@
 package main
 
 import (
+	"bytes"
+	"crypto/sha256"
 	"encoding/hex"
 	"errors"
+	"fmt"
+	"go.dedis.ch/cothority/v3/byzcoin"
+	"go.dedis.ch/onet/v3"
 	"io/ioutil"
 	"os"
 	"path"
 	"reflect"
 	"regexp"
 	"strings"
+	"time"
 
 	"go.dedis.ch/cothority/v3/skipchain"
 
@@ -120,6 +126,29 @@ func main() {
 			Action:    run,
 			ArgsUsage: "data-dir",
 		},
+		{
+			Name:      "proxy",
+			Aliases:   []string{"p"},
+			Usage:     "Act as a proxy to the real nodes",
+			Action:    runProxy,
+			ArgsUsage: "db-file",
+			Flags: []cli.Flag{
+				cli.IntFlag{
+					Name:  "portWS, p",
+					Value: 7771,
+					Usage: "port-number of the websocket",
+				},
+				cli.StringFlag{
+					Name:  "update, u",
+					Value: "1m",
+					Usage: "update interval",
+				},
+				cli.BoolFlag{
+					Name:  "complete",
+					Usage: "whether to fetch the complete chain",
+				},
+			},
+		},
 	}
 	cliApp.Flags = []cli.Flag{
 		cli.IntFlag{
@@ -147,6 +176,8 @@ func main() {
 	log.ErrFatal(err)
 }
 
+// configure creates a new private.toml from the command-line arguments.
+// It has some sensible defaults for some of the arguments.
 func configure(c *cli.Context) error {
 	conf := &app.CothorityConfig{Suite: cothority.Suite.String()}
 	dd := c.String("data-dir")
@@ -260,6 +291,7 @@ func configure(c *cli.Context) error {
 	return nil
 }
 
+// showConfig prints the private.toml in a human readable form.
 func showConfig(c *cli.Context) error {
 	if c.NArg() != 1 {
 		return xerrors.New("Please give data-dir")
@@ -287,6 +319,7 @@ func showConfig(c *cli.Context) error {
 	return nil
 }
 
+// run starts the node as a server, given a private.toml.
 func run(c *cli.Context) error {
 	if c.NArg() != 1 {
 		return xerrors.New("Please give data-dir")
@@ -299,3 +332,183 @@ func run(c *cli.Context) error {
 	server.Start()
 	return xerrors.New("server stopped unexpectedly")
 }
+
+// runProxy starts only a proxy that is not part of the network,
+// but merely fetches new blocks from the other nodes.
+// You can use this to have a fast node locally.
+// All security assumptions are still given,
+// as the forward-links will prove that the new blocks are correct.
+// It doesn't need to have a private.toml,
+// but merely uses a db with at least one chain defined.
+func runProxy(c *cli.Context) error {
+	if c.NArg() != 1 {
+		return xerrors.New("Please give database")
+	}
+
+	dur, err := time.ParseDuration(c.String("update"))
+	if err != nil {
+		return xerrors.Errorf("couldn't parse update: %v", err)
+	}
+
+	// Create a dummy ServerIdentity for the server to run.
+	addr := network.Address(fmt.Sprintf("tls://localhost:%d",
+		c.Int("portWS")-1))
+	privKey := cothority.Suite.Scalar().One()
+	pubKey := cothority.Suite.Point().Mul(privKey, nil)
+	si := network.NewServerIdentity(pubKey, addr)
+	si.SetPrivate(privKey)
+	services := make(map[string]app.ServiceConfig)
+	for _, name := range onet.ServiceFactory.RegisteredServiceNames() {
+		serviceSuite := onet.ServiceFactory.Suite(name)
+		if serviceSuite != nil {
+			kp := key.NewKeyPair(serviceSuite)
+			var pubBuf bytes.Buffer
+			if err := encoding.WriteHexPoint(serviceSuite, &pubBuf,
+				kp.Public); err != nil{
+				return xerrors.Errorf("couldn't marshal point: %v", err)
+			}
+			services[name] = app.ServiceConfig{Suite: serviceSuite.String(),
+				Public: pubBuf.String()}
+			si.ServiceIdentities = append(si.ServiceIdentities,
+				network.NewServiceIdentity(name, serviceSuite, kp.Public,
+					kp.Private))
+		}
+	}
+
+	serverToml := app.NewServerToml(cothority.Suite, pubKey, addr, "Proxy Server",
+		services)
+	if serverToml == nil{
+		return xerrors.New("unknown error when creating NewServerToml")
+	}
+	publicToml := app.NewGroupToml(serverToml)
+	log.Infof("public.toml:\n%s", publicToml.String())
+
+	// HACK: as we cannot give the db-name to the onet.Server structure,
+	// this creates a link in the /tmp-directory that points to the db given
+	// on the command-line.
+	pub, _ := pubKey.MarshalBinary()
+	dbName := fmt.Sprintf("%x.db", sha256.Sum256(pub))
+	if err := os.Setenv("CONODE_SERVICE_PATH", "/tmp"); err != nil {
+		return xerrors.Errorf("couldn't set CONODE_SERVICE_PATH: %v", err)
+	}
+
+	dbPath := path.Join("/tmp", dbName)
+	// Ignore the error of the link not found
+	_ = os.Remove(dbPath)
+	argFullPath := c.Args().First()
+	if !path.IsAbs(argFullPath) {
+		wd, err := os.Getwd()
+		if err != nil {
+			return xerrors.Errorf("couldn't get working directory: %v", err)
+		}
+		argFullPath = path.Join(wd, argFullPath)
+	}
+	if err := os.Symlink(argFullPath, dbPath); err != nil {
+		return xerrors.Errorf("couldn't create a symlink: %v", err)
+	}
+
+	// Finally get the server.
+	server := onet.NewServerTCP(si, cothority.Suite)
+
+	// Update the skipchains on a regular basis.
+	// This will also take care to update the global state.
+	go func() {
+		server.WaitStartup()
+		byzcoinService := server.Service(byzcoin.ServiceName).(*byzcoin.Service)
+
+		// Fetch all missing blocks if 'complete' is given.
+		if c.Bool("complete") {
+			err := updateSkipchains(server.Service(skipchain.ServiceName).(*skipchain.Service))
+			if err != nil {
+				log.Errorf("Couldn't update all blocks: %+v", err)
+			}
+		}
+
+		// To get the latest blocks, restart the service.
+		// This might create new holes in the chain if there are too many
+		// blocks created between two calls.
+		for {
+			time.Sleep(dur)
+			log.Info("Fetching new blocks")
+			byzcoinService.TestClose()
+			if err := byzcoinService.TestRestart(); err != nil {
+				log.Fatalf("Couldn't restart service: %+v", err)
+			}
+		}
+	}()
+
+	server.Start()
+	return xerrors.New("server stopped unexpectedly")
+}
+
+// updateSkipchains does what skipchain.Service.
+// DoSync doesn't do: it fills all holes in the skipchain.
+func updateSkipchains(scs *skipchain.Service) error {
+	log.Info("Searching and downloading blocks for all chains")
+	scIDs, err := scs.GetDB().GetSkipchains()
+	if err != nil {
+		return xerrors.Errorf("couldn't fetch all skipchain-IDs: %v", err)
+	} else {
+	chains:
+		for sc, latest := range scIDs {
+			scID := skipchain.SkipBlockID(sc)
+			log.Lvlf1("Checking chain %x up to block %d", scID[:], latest.Index)
+			current := scs.GetDB().GetByID(scID)
+			cl := skipchain.NewClient()
+			for {
+				if current.Index%10000 == 0 {
+					log.Lvlf1("Scanning block %d/%d", current.Index,
+						latest.Index)
+				}
+				var next *skipchain.SkipBlock
+				if len(current.ForwardLink) == 0 {
+					// latest might already be outdated once we get here,
+					// in the case where the skipchain-service found a later
+					// block.
+					latest, err = scs.GetDB().GetLatestByID(scID)
+					if err != nil {
+						return xerrors.Errorf(
+							"couldn't get latest block: %v", err)
+					}
+					if current.Hash.Equal(latest.Hash) {
+						log.Lvlf1("Done with chain %x at block %d",
+							scID[:], current.Index)
+						return nil
+					}
+					log.Lvlf1("Found premature end of chain - downloading hole")
+				} else {
+					next = scs.GetDB().GetByID(current.ForwardLink[0].To)
+				}
+
+				if next != nil {
+					current = next
+				} else {
+					log.Lvlf1("Downloading blocks at index %d", current.Index)
+					blocks, err := cl.GetUpdateChainLevel(latest.Roster,
+						current.Hash, 1, 1000)
+					if err != nil {
+						return xerrors.Errorf(
+							"error while getting blocks: %v", err)
+					}
+					if len(blocks) < 2 {
+						log.Warn("Couldn't get new blocks")
+						continue chains
+					}
+					log.Lvlf1("Got %d new blocks", len(blocks))
+					_, err = scs.GetDB().StoreBlocks(blocks)
+					if err != nil {
+						return xerrors.Errorf("couldn't store blocks: %v", err)
+					}
+					for _, sb := range blocks[1:] {
+						if sb.Index == current.Index+1 {
+							current = sb
+						} else {
+							break
+						}
+					}
+				}
+			}
+		}
+	}
+	return nil
+}