diff --git a/simul/README.md b/simul/README.md index 3bc6df99a..6a06fb024 100644 --- a/simul/README.md +++ b/simul/README.md @@ -34,6 +34,14 @@ If you use the `onet.SimulationBFTree`, the following variables are also availab - Depth - the depth of the tree in levels below the root-node - Rounds - for how many rounds the simulation should run +### Simulations with long setup-times and multiple measurements + +Per default, all rounds of an individual simulation-run will be averaged and +written to the csv-file. If you set `IndividualStats` to a non-`""`-value, +every round will create a new line. This is useful if you have a simulation +with a long setup-time and you want to do multiple measurements for the same +setup. + ### Timeouts Two timeout variables are available: diff --git a/simul/build.go b/simul/build.go index d9489cc75..60ff0b1b1 100644 --- a/simul/build.go +++ b/simul/build.go @@ -114,10 +114,6 @@ func RunTests(name string, runconfigs []*platform.RunConfig) { } mkTestDir() - rs := make([]*monitor.Stats, len(runconfigs)) - // Try 10 times to run the test - nTimes := 10 - stopOnSuccess := true var f *os.File args := os.O_CREATE | os.O_RDWR | os.O_TRUNC // If a range is given, we only append @@ -150,31 +146,21 @@ func RunTests(name string, runconfigs []*platform.RunConfig) { // run test t nTimes times // take the average of all successful runs - runs := make([]*monitor.Stats, 0, nTimes) - for r := 0; r < nTimes; r++ { - stats, err := RunTest(rc) - if err != nil { - log.Error("Error running test, trying again:", err) - continue - } - - runs = append(runs, stats) - if stopOnSuccess { - break - } - } - - if len(runs) == 0 { - log.Lvl1("unable to get any data for test:", rc) + stats, err := RunTest(rc) + if err != nil { + log.Error("Error running test, trying again:", err) continue } - s := monitor.AverageStats(runs) if i == 0 { - s.WriteHeader(f) + stats.WriteHeader(f) + } + if rc.Get("IndividualStats") != "" { + err := stats.WriteIndividualStats(f) + log.ErrFatal(err) + } else { + stats.WriteValues(f) } - rs[i] = s - rs[i].WriteValues(f) err = f.Sync() if err != nil { log.Fatal("error syncing data to test file:", err) diff --git a/simul/manage/simulation/individualstats.toml b/simul/manage/simulation/individualstats.toml new file mode 100644 index 000000000..2ab452bb1 --- /dev/null +++ b/simul/manage/simulation/individualstats.toml @@ -0,0 +1,7 @@ +Servers = 16 +Simulation = "Count" +BF = 2 +IndividualStats = "true" + +Hosts,Rounds +3, 5 diff --git a/simul/manage/simulation/simul_test.go b/simul/manage/simulation/simul_test.go index 28c1711d8..06503202e 100644 --- a/simul/manage/simulation/simul_test.go +++ b/simul/manage/simulation/simul_test.go @@ -3,9 +3,29 @@ package main import ( "testing" + "io/ioutil" + + "strings" + "github.com/dedis/onet/simul" + "github.com/stretchr/testify/assert" + "gopkg.in/dedis/onet.v1/log" ) func TestSimulation(t *testing.T) { simul.Start("count.toml", "csv1.toml", "csv2.toml") } + +func TestSimulation_IndividualStats(t *testing.T) { + simul.Start("individualstats.toml") + csv, err := ioutil.ReadFile("test_data/individualstats.csv") + log.ErrFatal(err) + // header + 5 rounds + final newline + assert.Equal(t, 7, len(strings.Split(string(csv), "\n"))) + + simul.Start("csv1.toml") + csv, err = ioutil.ReadFile("test_data/csv1.csv") + log.ErrFatal(err) + // header + 2 experiments + final newline + assert.Equal(t, 4, len(strings.Split(string(csv), "\n"))) +} diff --git a/simul/monitor/stats.go b/simul/monitor/stats.go index af671f78c..ee983f53b 100644 --- a/simul/monitor/stats.go +++ b/simul/monitor/stats.go @@ -10,6 +10,8 @@ import ( "strings" "sync" + "errors" + "github.com/dedis/onet/log" "github.com/montanaflynn/stats" ) @@ -111,6 +113,59 @@ func (s *Stats) WriteValues(w io.Writer) { fmt.Fprintf(w, "\n") } +// WriteIndividualStats will write the values to the specified writer but without +// making averages. Each value should either be: +// - represented once - then it'll be copied to all runs +// - have the same frequency as the other non-once values +func (s *Stats) WriteIndividualStats(w io.Writer) error { + // by default + s.Lock() + defer s.Unlock() + + // Verify we have either one or n values, where n >= 1 but constant + // over all values + n := 1 + for _, k := range s.keys { + if newN := len(s.values[k].store); newN > 1 { + if n == 1 { + n = newN + } else if n != newN { + return errors.New("Found inconsistencies in values") + } + } + } + + // store static fields + var static []string + for _, k := range s.staticKeys { + if v, ok := s.static[k]; ok { + static = append(static, fmt.Sprintf("%d", v)) + } + } + + // add all values + for entry := 0; entry < n; entry++ { + var values []string + // write the values + for _, k := range s.keys { + v := s.values[k] + values = append(values, v.SingleValues(entry)...) + } + + all := append(static, values...) + _, err := fmt.Fprintf(w, "%s", strings.Join(all, ",")) + if err != nil { + return err + } + _, err = fmt.Fprintf(w, "\n") + if err != nil { + return err + } + + } + return nil +} + // AverageStats will make an average of the given stats func AverageStats(stats []*Stats) *Stats { if len(stats) < 1 { @@ -216,7 +271,7 @@ func (df *DataFilter) Filter(measure string, values []float64) []float64 { } // Collect make the final computations before stringing or writing. -// Autmatically done in other methods anyway. +// Automatically done in other methods anyway. func (s *Stats) Collect() { s.Lock() defer s.Unlock() @@ -447,3 +502,12 @@ func (t *Value) HeaderFields() []string { func (t *Value) Values() []string { return []string{fmt.Sprintf("%f", t.Min()), fmt.Sprintf("%f", t.Max()), fmt.Sprintf("%f", t.Avg()), fmt.Sprintf("%f", t.Sum()), fmt.Sprintf("%f", t.Dev())} } + +// SingleValues returns the string representation of an entry in the value +func (t *Value) SingleValues(i int) []string { + v := fmt.Sprintf("%f", t.store[0]) + if i < len(t.store) { + v = fmt.Sprintf("%f", t.store[i]) + } + return []string{v, v, v, v, "NaN"} +} diff --git a/simul/platform/runsimul.go b/simul/platform/runsimul.go index 8cbd7171a..a96fd64e2 100644 --- a/simul/platform/runsimul.go +++ b/simul/platform/runsimul.go @@ -6,6 +6,7 @@ import ( "github.com/dedis/onet" "github.com/dedis/onet/log" + "github.com/BurntSushi/toml" "github.com/dedis/onet/network" "github.com/dedis/onet/simul/manage" "github.com/dedis/onet/simul/monitor" @@ -38,20 +39,33 @@ func Simulate(serverAddress, simul, monitorAddress string) error { // having a waitgroup so the binary stops when all servers are closed var wgServer, wgSimulInit sync.WaitGroup var ready = make(chan bool) + measureNodeBW := true + if len(scs) > 0 { + cfg := &conf{} + _, err := toml.Decode(scs[0].Config, cfg) + if err != nil { + return err + } + measureNodeBW = cfg.IndividualStats == "" + } for i, sc := range scs { // Starting all servers for that server server := sc.Server - measures[i] = monitor.NewCounterIOMeasure("bandwidth", server) log.Lvl3(serverAddress, "Starting server", server.ServerIdentity.Address) + if measureNodeBW { + measures[i] = monitor.NewCounterIOMeasure("bandwidth", server) + } // Launch a server and notifies when it's done - wgServer.Add(1) go func(c *onet.Server, m monitor.Measure) { ready <- true defer wgServer.Done() c.Start() - // record bandwidth - m.Record() + // record bandwidth, except if we're measuring every + // round individually + if measureNodeBW { + m.Record() + } log.Lvl3(serverAddress, "Simulation closed server", c.ServerIdentity) }(server, measures[i]) // wait to be sure the goroutine started @@ -161,3 +175,7 @@ func Simulate(serverAddress, simul, monitorAddress string) error { } return nil } + +type conf struct { + IndividualStats string +}