database investigation (#5)

* add readme documentation

* initial skeleton

* create DB

* insert many rows at once

* table with varchar(32) as index

* first select query

* run query with index

* more queries

* query with prepared stmt

* queries with different IDs, to avoid always hitting cache

* query with multiple threads

* query with random ids,multiple threads

* preserving some sql for now

* read random IDs from DB

* multiple connections

* more benchmark results for multiple connections

* updateStructure and test

* fix bug (store a copy of leaf ID).

The tree was storing the original pointers to the leaf IDs passed
from the test. When these values were modified, the ID inside the
nodes of the tree was modified. Fixed it.

* wip explicit structure

* wip 2 explicit structure

* insert with explicit structure

* refactoring

* refactoring

* select leaves with explicit structure

* WIP stored proc

* stored function

* fix get nodes stored procedure.

When the nodes is not found, return an empty blob.

* fix minor display hint for the experiments

* run new experiments at more powerful machine

* move db_test inside cmd/

* clean up

Schema, table and stored proc/fcn creation.
Connection creation.

* define and use Conn interface.

Examples of its use.
Querying functions.
Table schema contains now a proof per node.

* last results, stored proc

Author: juagargi

cmd/db_test/main.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/netsec-ethz/fpki/pkg/db"
+)
+
+func main() {
+	truncateFlag := flag.Bool("truncate", false, "insert values")
+	insertFlag := flag.Bool("insert", false, "insert values")
+	queryFlag := flag.Bool("query", false, "perform a query")
+	flag.Parse()
+
+	config := db.Configuration{
+		Dsn: "root@tcp(localhost)/fpki",
+		Values: map[string]string{
+			"interpolateParams": "true", // 1 round trip per query
+			"collation":         "binary",
+		},
+	}
+	createConn := func() (db.Conn, error) { // define a function that creates connections
+		return db.Connect(&config)
+	}
+
+	var err error
+	t0 := time.Now()
+
+	if *truncateFlag {
+		c, err := createConn()
+		check(err)
+		err = db.DeletemeDropAllNodes(c)
+		check(err)
+		t0 = time.Now()
+	}
+	if *insertFlag {
+		c, err := createConn()
+		check(err)
+		err = db.DeletemeCreateNodes2(c, 100*1000)
+		check(err)
+	}
+	if *queryFlag {
+		// t0, err = db.DeletemeSelectLeavesStoredFunc3(createConn, 100*1000+96, 8, 8) // 3.085243872s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc3(createConn, 100*1000+96, 16, 1) // 4.289929758s
+		t0, err = db.DeletemeSelectLeavesStoredFunc3(createConn, 100*1000+96, 1, 16) // 4.305125693s
+		check(err)
+	}
+	fmt.Printf("time: %s\n", time.Since(t0))
+	fmt.Println("ready")
+}
+
+func main2() {
+	truncateFlag := flag.Bool("truncate", false, "insert values")
+	insertFlag := flag.Bool("insert", false, "insert values")
+	queryFlag := flag.Bool("query", false, "perform a query")
+	flag.Parse()
+
+	c, err := db.Connect_old()
+	check(err)
+	defer c.Close()
+
+	t0 := time.Now()
+
+	if *truncateFlag {
+		err = db.DeletemeDropAllNodes(c)
+		check(err)
+		t0 = time.Now()
+	}
+	if *insertFlag {
+		// err = db.DeletemeCreateNodes(c, 1000) // 5.975667276s
+
+		// err = db.DeletemeCreateNodesBulk(c, 1000) // 60.954469ms
+		// err = db.DeletemeCreateNodesBulk(c, 1000000) // too slow (before prepared statements)
+		// err = db.DeletemeCreateNodesBulk(c, 50000) // 1.724559003s
+		// err = db.DeletemeCreateNodesBulk(c, 100000) // too slow
+
+		// err = db.DeletemeCreateNodesBulk2(c, 200000) // 1.161974957s
+		// err = db.DeletemeCreateNodesBulk2(c, 1000*1000) // 11.807568854s
+
+		// err = db.DeletemeCreateNodesBulk3(c, 1000*1000) // 10.23273209s
+
+		// err = db.DeletemeCreateNodesBulk4(c, 100*1000) // 1.693980634s
+		// err = db.DeletemeCreateNodesBulk4(c, 1000*1000) // 17.182428746s
+
+		// err = db.DeletemeCreateNodes2(c, 1000) // 212.752302ms
+		// err = db.DeletemeCreateNodes2(c, 10*1000) // 456.4187ms
+		err = db.DeletemeCreateNodes2(c, 100*1000) // 9.51899883s
+		// err = db.DeletemeCreateNodes2(c, 100*1000) // 4.351831675s
+		// err = db.DeletemeCreateNodes2(c, 1000*1000) // 1m16.537726013s
+	}
+	if *queryFlag {
+		// err = db.DeletemeSelectNodes(c, 1) // 975.161µs
+		// err = db.DeletemeSelectNodes(c, 1000) // 108.110884ms
+		// err = db.DeletemeSelectNodes(c, 100*1000) // 10.606129185s
+
+		// err = db.DeletemeSelectNodes2(c, 100*1000) // 8.854317991s
+
+		// err = db.DeletemeSelectNodes3(c, 100*1000, 8) // 2.184544624s
+		// err = db.DeletemeSelectNodes3(c, 500*1000, 32) // 11.123422821s = 1M in 22.2s
+
+		// err = db.DeletemeSelectNodesRandom4(c, 100*1000, 32) // 2.176723473s
+		// err = db.DeletemeSelectNodesRandom4(c, 1000*1000, 16) // 16.294332418s
+		// err = db.DeletemeSelectNodesRandom4(c, 1000*1000, 32) // 16.207375911s
+		// err = db.DeletemeSelectNodesRandom4(c, 1000*1000, 64) // 16.208407496s
+
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000, 16, 2) // 7.190813246s
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000, 32, 2) // 6.906053331s
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000+64, 32, 4) // 5.78090823s
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000+64, 64, 2) // 5.845077626s
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000+192, 64, 4) // 5.57853586s
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000+192, 128, 1) // 5.772750195s
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000+192, 128, 2) // 5.593609741s
+		// t0, err = db.DeletemeSelectNodesRandom5(1000*1000+448, 128, 4) // 5.829905885s
+
+		// t0, err = db.DeletemeSelectLeaves(100) // 164.934957ms
+		// t0, err = db.DeletemeSelectLeaves(1000) // 1.446072294s
+		// t0, err = db.DeletemeSelectLeaves(10 * 1000) // 14.24444017s
+
+		// t0, err = db.DeletemeSelectLeavesStoredProc(1000) // 608.293495ms
+
+		// t0, err = db.DeletemeSelectLeavesStoredFunc(1000) // 459.793285ms
+		// t0, err = db.DeletemeSelectLeavesStoredFunc(100 * 1000) // 36.995034654s
+
+		// values at ETH's computer
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(1000, 1, 1) // 373.195752ms
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(1000, 1, 8) // 95.495026ms
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000, 1, 8) // 8.746812923s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000, 1, 16) // 8.096993868s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000, 8, 1) // 8.737253918s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000+32, 8, 8) // 7.914086447s
+
+		// values at home computer:
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(1000, 1, 1) // 244.728829ms
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(1000, 1, 8) // 55.34648ms
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000, 1, 8) // 4.77145546s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000, 1, 16) // 3.158051082s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000, 8, 1) // 4.794854328s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(100*1000+32, 8, 8) // 2.429056914s
+		// t0, err = db.DeletemeSelectLeavesStoredFunc2(1000*1000+192, 16, 16) // 21.987717762s
+	}
+
+	check(err)
+	fmt.Printf("time: %s\n", time.Since(t0))
+	fmt.Println("ready")
+}
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}

go.mod

@@ -0,0 +1,14 @@
+module github.com/netsec-ethz/fpki
+
+go 1.17
+
+require (
+	github.com/go-sql-driver/mysql v1.6.0
+	github.com/stretchr/testify v1.7.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)

go.sum

@@ -0,0 +1,13 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/db/db.go

@@ -0,0 +1,16 @@
+package db
+
+import "context"
+
+type FullID [33]byte // first byte is depth -1 (root not allowed)
+
+type Conn interface {
+	// Close closes the connection.
+	Close() error
+	// RetrieveValue returns the value associated with the node.
+	RetrieveValue(ctx context.Context, id FullID) ([]byte, error)
+	// RetrieveNode returns the value and the proof path (without the root) for a given node.
+	// Since each one of the steps of the proof path has a fixed size, returning the path
+	// as a slice is sufficient to know how many steps there were in the proof path.
+	RetrieveNode(ctx context.Context, id FullID) ([]byte, []byte, error)
+}