adding flag to print every round-stat instead of averageing them (#172)

* adding flag to print every round-stat instead of averageing them

* renamed IndividualStats and made it work with bandwidth

simul/README.md

@@ -34,6 +34,14 @@ If you use the `onet.SimulationBFTree`, the following variables are also availab
 - Depth - the depth of the tree in levels below the root-node
 - Rounds - for how many rounds the simulation should run
 
+### Simulations with long setup-times and multiple measurements
+
+Per default, all rounds of an individual simulation-run will be averaged and
+written to the csv-file. If you set `IndividualStats` to a non-`""`-value,
+every round will create a new line. This is useful if you have a simulation
+with a long setup-time and you want to do multiple measurements for the same
+setup.
+
 ### Timeouts
 
 Two timeout variables are available:

simul/build.go

@@ -114,10 +114,6 @@ func RunTests(name string, runconfigs []*platform.RunConfig) {
 	}
 
 	mkTestDir()
-	rs := make([]*monitor.Stats, len(runconfigs))
-	// Try 10 times to run the test
-	nTimes := 10
-	stopOnSuccess := true
 	var f *os.File
 	args := os.O_CREATE | os.O_RDWR | os.O_TRUNC
 	// If a range is given, we only append
@@ -150,31 +146,21 @@ func RunTests(name string, runconfigs []*platform.RunConfig) {
 
 		// run test t nTimes times
 		// take the average of all successful runs
-		runs := make([]*monitor.Stats, 0, nTimes)
-		for r := 0; r < nTimes; r++ {
-			stats, err := RunTest(rc)
-			if err != nil {
-				log.Error("Error running test, trying again:", err)
-				continue
-			}
-
-			runs = append(runs, stats)
-			if stopOnSuccess {
-				break
-			}
-		}
-
-		if len(runs) == 0 {
-			log.Lvl1("unable to get any data for test:", rc)
+		stats, err := RunTest(rc)
+		if err != nil {
+			log.Error("Error running test, trying again:", err)
 			continue
 		}
 
-		s := monitor.AverageStats(runs)
 		if i == 0 {
-			s.WriteHeader(f)
+			stats.WriteHeader(f)
+		}
+		if rc.Get("IndividualStats") != "" {
+			err := stats.WriteIndividualStats(f)
+			log.ErrFatal(err)
+		} else {
+			stats.WriteValues(f)
 		}
-		rs[i] = s
-		rs[i].WriteValues(f)
 		err = f.Sync()
 		if err != nil {
 			log.Fatal("error syncing data to test file:", err)

simul/manage/simulation/individualstats.toml

@@ -0,0 +1,7 @@
+Servers = 16
+Simulation = "Count"
+BF = 2
+IndividualStats = "true"
+
+Hosts,Rounds
+3, 5

simul/manage/simulation/simul_test.go

@@ -3,9 +3,29 @@ package main
 import (
 	"testing"
 
+	"io/ioutil"
+
+	"strings"
+
 	"github.com/dedis/onet/simul"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/dedis/onet.v1/log"
 )
 
 func TestSimulation(t *testing.T) {
 	simul.Start("count.toml", "csv1.toml", "csv2.toml")
 }
+
+func TestSimulation_IndividualStats(t *testing.T) {
+	simul.Start("individualstats.toml")
+	csv, err := ioutil.ReadFile("test_data/individualstats.csv")
+	log.ErrFatal(err)
+	// header + 5 rounds + final newline
+	assert.Equal(t, 7, len(strings.Split(string(csv), "\n")))
+
+	simul.Start("csv1.toml")
+	csv, err = ioutil.ReadFile("test_data/csv1.csv")
+	log.ErrFatal(err)
+	// header + 2 experiments + final newline
+	assert.Equal(t, 4, len(strings.Split(string(csv), "\n")))
+}

simul/monitor/stats.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"sync"
 
+	"errors"
+
 	"github.com/dedis/onet/log"
 	"github.com/montanaflynn/stats"
 )
@@ -111,6 +113,59 @@ func (s *Stats) WriteValues(w io.Writer) {
 	fmt.Fprintf(w, "\n")
 }
 
+// WriteIndividualStats will write the values to the specified writer but without
+// making averages. Each value should either be:
+//   - represented once - then it'll be copied to all runs
+//   - have the same frequency as the other non-once values
+func (s *Stats) WriteIndividualStats(w io.Writer) error {
+	// by default
+	s.Lock()
+	defer s.Unlock()
+
+	// Verify we have either one or n values, where n >= 1 but constant
+	// over all values
+	n := 1
+	for _, k := range s.keys {
+		if newN := len(s.values[k].store); newN > 1 {
+			if n == 1 {
+				n = newN
+			} else if n != newN {
+				return errors.New("Found inconsistencies in values")
+			}
+		}
+	}
+
+	// store static fields
+	var static []string
+	for _, k := range s.staticKeys {
+		if v, ok := s.static[k]; ok {
+			static = append(static, fmt.Sprintf("%d", v))
+		}
+	}
+
+	// add all values
+	for entry := 0; entry < n; entry++ {
+		var values []string
+		// write the values
+		for _, k := range s.keys {
+			v := s.values[k]
+			values = append(values, v.SingleValues(entry)...)
+		}
+
+		all := append(static, values...)
+		_, err := fmt.Fprintf(w, "%s", strings.Join(all, ","))
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprintf(w, "\n")
+		if err != nil {
+			return err
+		}
+
+	}
+	return nil
+}
+
 // AverageStats will make an average of the given stats
 func AverageStats(stats []*Stats) *Stats {
 	if len(stats) < 1 {
@@ -216,7 +271,7 @@ func (df *DataFilter) Filter(measure string, values []float64) []float64 {
 }
 
 // Collect make the final computations before stringing or writing.
-// Autmatically done in other methods anyway.
+// Automatically done in other methods anyway.
 func (s *Stats) Collect() {
 	s.Lock()
 	defer s.Unlock()
@@ -447,3 +502,12 @@ func (t *Value) HeaderFields() []string {
 func (t *Value) Values() []string {
 	return []string{fmt.Sprintf("%f", t.Min()), fmt.Sprintf("%f", t.Max()), fmt.Sprintf("%f", t.Avg()), fmt.Sprintf("%f", t.Sum()), fmt.Sprintf("%f", t.Dev())}
 }
+
+// SingleValues returns the string representation of an entry in the value
+func (t *Value) SingleValues(i int) []string {
+	v := fmt.Sprintf("%f", t.store[0])
+	if i < len(t.store) {
+		v = fmt.Sprintf("%f", t.store[i])
+	}
+	return []string{v, v, v, v, "NaN"}
+}

simul/platform/runsimul.go

@@ -6,6 +6,7 @@ import (
 	"github.com/dedis/onet"
 	"github.com/dedis/onet/log"
 
+	"github.com/BurntSushi/toml"
 	"github.com/dedis/onet/network"
 	"github.com/dedis/onet/simul/manage"
 	"github.com/dedis/onet/simul/monitor"
@@ -38,20 +39,33 @@ func Simulate(serverAddress, simul, monitorAddress string) error {
 	// having a waitgroup so the binary stops when all servers are closed
 	var wgServer, wgSimulInit sync.WaitGroup
 	var ready = make(chan bool)
+	measureNodeBW := true
+	if len(scs) > 0 {
+		cfg := &conf{}
+		_, err := toml.Decode(scs[0].Config, cfg)
+		if err != nil {
+			return err
+		}
+		measureNodeBW = cfg.IndividualStats == ""
+	}
 	for i, sc := range scs {
 		// Starting all servers for that server
 		server := sc.Server
-		measures[i] = monitor.NewCounterIOMeasure("bandwidth", server)
 		log.Lvl3(serverAddress, "Starting server", server.ServerIdentity.Address)
+		if measureNodeBW {
+			measures[i] = monitor.NewCounterIOMeasure("bandwidth", server)
+		}
 		// Launch a server and notifies when it's done
-
 		wgServer.Add(1)
 		go func(c *onet.Server, m monitor.Measure) {
 			ready <- true
 			defer wgServer.Done()
 			c.Start()
-			// record bandwidth
-			m.Record()
+			// record bandwidth, except if we're measuring every
+			// round individually
+			if measureNodeBW {
+				m.Record()
+			}
 			log.Lvl3(serverAddress, "Simulation closed server", c.ServerIdentity)
 		}(server, measures[i])
 		// wait to be sure the goroutine started
@@ -161,3 +175,7 @@ func Simulate(serverAddress, simul, monitorAddress string) error {
 	}
 	return nil
 }
+
+type conf struct {
+	IndividualStats string
+}