Updated serializers to current versions (27-01-2020).

.gitignore

@@ -5,3 +5,6 @@ compile_commands.json
 build/
 .vscode/
 .clangd/
+.RData
+.Rhistory
+*.Rout

CMakeLists.txt

@@ -67,11 +67,11 @@ set(boost_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/boost)
 ExternalProject_Add(
     boost
     PREFIX ${boost_PREFIX}
-    URL "http://netcologne.dl.sourceforge.net/project/boost/boost/1.69.0/boost_1_69_0.tar.gz"
-    URL_MD5 "b50944c0c13f81ce2c006802a1186f5a"
-    CONFIGURE_COMMAND ${boost_PREFIX}/src/boost/bootstrap.sh --with-libraries=serialization,filesystem,system,program_options,iostreams --prefix=${boost_PREFIX}
-    BUILD_COMMAND ./bjam link=static cxxflags='-fPIC'
-    INSTALL_COMMAND ./bjam link=static install
+    URL "https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz"
+    URL_MD5 "e2b0b1eac302880461bcbef097171758"
+    CONFIGURE_COMMAND ${boost_PREFIX}/src/boost/bootstrap.sh --with-libraries=serialization,filesystem,system,program_options,iostreams,regex --prefix=${boost_PREFIX}
+    BUILD_COMMAND ./b2 link=static cxxflags='-fPIC'
+    INSTALL_COMMAND ./b2 link=static install
     BUILD_IN_SOURCE 1
     LOG_UPDATE ON
     LOG_CONFIGURE ON
@@ -87,8 +87,8 @@ ExternalProject_Add(
     thrift
     PREFIX ${thrift_PREFIX}
     DEPENDS boost
-    URL "http://www-eu.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz"
-    URL_MD5 "3deebbb4d1ca77dd9c9e009a1ea02183"
+    URL "https://www-eu.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz"
+    URL_MD5 "38a27d391a2b03214b444cb13d5664f1"
     CONFIGURE_COMMAND CXX=${CMAKE_CXX_COMPILER} CC=${CMAKE_C_COMPILER} ${thrift_PREFIX}/src/thrift/configure --prefix=${thrift_PREFIX} --libdir=${thrift_PREFIX}/lib --with-boost=${Boost_INCLUDE_DIRS} --enable-shared=no --with-libevent=no --with-c_glib=no --with-java=no --with-erlang=no --with-python=no --with-perl=no --with-php=no --with-php_extension=no --with-ruby=no --with-haskell=no --with-go=no --with-d=no --with-lua=no --with-qt4=no --with-qt5=no --with-nodejs=no --with-cl=no --with-dotnetcore=no --with-nodets=no --with-rs=no --disable-tests --enable-plugin=no
     BUILD_COMMAND $(MAKE)
     INSTALL_COMMAND $(MAKE) install
@@ -118,8 +118,8 @@ set(protobuf_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/protobuf)
 ExternalProject_Add(
     protobuf
     PREFIX ${protobuf_PREFIX}
-    URL "https://github.com/protocolbuffers/protobuf/releases/download/v3.7.0/protobuf-cpp-3.7.0.tar.gz"
-    URL_MD5 "f1631a8e551e569273d78538f6ecf41c"
+    URL "https://github.com/protocolbuffers/protobuf/releases/download/v3.11.2/protobuf-cpp-3.11.2.tar.gz"
+    URL_MD5 "351a0f5c4edd3f45035417f09133ce73"
     CONFIGURE_COMMAND CXX=${CMAKE_CXX_COMPILER} CC=${CMAKE_C_COMPILER} ${protobuf_PREFIX}/src/protobuf/configure --prefix=${protobuf_PREFIX} --libdir=${protobuf_PREFIX}/lib --enable-shared=no
     BUILD_COMMAND $(MAKE)
     INSTALL_COMMAND $(MAKE) install
@@ -153,8 +153,8 @@ set(cereal_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/cereal)
 ExternalProject_Add(
     cereal
     PREFIX ${cereal_PREFIX}
-    URL "https://github.com/USCiLab/cereal/archive/v1.2.2.tar.gz"
-    URL_MD5 "4c56c7b9499dba79404250ef9a040481"
+    URL "https://github.com/USCiLab/cereal/archive/v1.3.0.tar.gz"
+    URL_MD5 "4342e811f245403646c4175258f413f1"
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
     INSTALL_COMMAND mkdir -p ${cereal_PREFIX}/include/ && cp -r ${cereal_PREFIX}/src/cereal/include/cereal ${cereal_PREFIX}/include/
@@ -165,8 +165,8 @@ ExternalProject_Add(
     avro
     DEPENDS boost
     PREFIX ${avro_PREFIX}
-    URL "https://github.com/apache/avro/archive/release-1.8.2.tar.gz"
-    URL_MD5 "dcaa6eaa0c0854c535b9daf1c024cb35"
+    URL "https://github.com/apache/avro/archive/release-1.9.1.tar.gz"
+    URL_MD5 "2929b6ee91070fe0f2f6e683c7892104"
     CONFIGURE_COMMAND cmake -DBOOST_INCLUDEDIR=${boost_PREFIX}/include -DBOOST_LIBRARYDIR=${boost_PREFIX}/lib -DBoost_NO_SYSTEM_PATHS=ON -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${avro_PREFIX} -DBUILD_SHARED_LIBS=OFF ${avro_PREFIX}/src/avro/lang/c++/
     # Due to some issues with using statically built boost libraries AND zlib in avro tests
     # build only necessary parts and install them in a very hackish way.
@@ -183,8 +183,8 @@ set(flatbuffers_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/flatbuffers)
 ExternalProject_Add(
     flatbuffers
     PREFIX ${flatbuffers_PREFIX}
-    URL "https://github.com/google/flatbuffers/archive/v1.10.0.tar.gz"
-    URL_MD5 "f7d19a3f021d93422b0bc287d7148cd2"
+    URL "https://github.com/google/flatbuffers/archive/v1.11.0.tar.gz"
+    URL_MD5 "02c64880acb89dbd57eebacfd67200d8"
     CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${flatbuffers_PREFIX} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     LOG_UPDATE ON
     LOG_CONFIGURE ON
@@ -197,8 +197,8 @@ set(yas_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/yas)
 ExternalProject_Add(
     yas
     PREFIX ${yas_PREFIX}
-    URL "https://github.com/niXman/yas/archive/7.0.2.tar.gz"
-    URL_MD5 "d55353960467afabc6774583880a30f0"
+    URL "https://github.com/niXman/yas/archive/7.0.5.tar.gz"
+    URL_MD5 "8c666d29575c8cafe9cea1fc2946253e"
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
     INSTALL_COMMAND mkdir -p ${yas_PREFIX}/include/ && cp -r ${yas_PREFIX}/src/yas/include/yas ${yas_PREFIX}/include/

avro/record.hpp

@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *     https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,8 +17,8 @@
  */
 
 
-#ifndef _HOME_PKOK_CPP_SERIALIZERS_AVRO_RECORD_HPP_2710640864__H_
-#define _HOME_PKOK_CPP_SERIALIZERS_AVRO_RECORD_HPP_2710640864__H_
+#ifndef _HOME_PKOK_DROPBOX_UNIVERSITY_THESIS_CPP_SERIALIZERS_AVRO_RECORD_HPP_2093218524__H_
+#define _HOME_PKOK_DROPBOX_UNIVERSITY_THESIS_CPP_SERIALIZERS_AVRO_RECORD_HPP_2093218524__H_
 
 
 #include <sstream>

flatbuffers/test_generated.h

@@ -11,7 +11,7 @@ namespace flatbuffers_test {
 struct Record;
 
 struct Record FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
-  enum {
+  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_IDS = 4,
     VT_STRINGS = 6
   };
@@ -67,10 +67,12 @@ inline flatbuffers::Offset<Record> CreateRecordDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
     const std::vector<int64_t> *ids = nullptr,
     const std::vector<flatbuffers::Offset<flatbuffers::String>> *strings = nullptr) {
+  auto ids__ = ids ? _fbb.CreateVector<int64_t>(*ids) : 0;
+  auto strings__ = strings ? _fbb.CreateVector<flatbuffers::Offset<flatbuffers::String>>(*strings) : 0;
   return flatbuffers_test::CreateRecord(
       _fbb,
-      ids ? _fbb.CreateVector<int64_t>(*ids) : 0,
-      strings ? _fbb.CreateVector<flatbuffers::Offset<flatbuffers::String>>(*strings) : 0);
+      ids__,
+      strings__);
 }
 
 inline const flatbuffers_test::Record *GetRecord(const void *buf) {

images/generate_graph_data.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+ITERATIONS=1000000
+RESTARTS=50
+BENCHMARK="../build/benchmark"
+
+if [ ! -f ${BENCHMARK} ]; then
+  echo "Error: executable build/benchmark not found"
+  exit 1
+fi
+
+echo "-- Generating data for images/graphs.R..."
+echo "-- Substitute lines 9 through 19 with this:"
+c=" "
+for t in thrift-binary thrift-compact protobuf boost msgpack cereal avro capnproto flatbuffers yas yas-compact;
+do
+  ${BENCHMARK} -i 1 -s $t | grep Size | awk "{printf \"%s%d # %s\n\", \"$c\", \$3, \"$t\"}"
+  c=","
+done
+
+echo ""
+echo "-- Substitute lines 23 through 31 with this:"
+c=" "
+for t in thrift-binary thrift-compact protobuf boost msgpack cereal avro yas yas-compact;
+do
+  rm -f /tmp/$t.time
+  for i in `seq 1 ${RESTARTS}`
+  do 
+    ${BENCHMARK} -i ${ITERATIONS} -s $t | grep Time | awk '{print $3}' >>/tmp/$t.time
+  done;
+  awk "{ sum += \$1 } END {printf \"%s%f # %s\n\", \"$c\", sum/50, \"$t\"}" /tmp/$t.time
+  c=","
+done
+
+echo ""
+echo "-- Substitute lines 35 and 36 with this:"
+c=" "
+for t in capnproto flatbuffers;
+do
+  rm -f /tmp/$t.time;
+  for i in `seq 1 ${RESTARTS}`;
+  do 
+    ${BENCHMARK} -i ${ITERATIONS} -s $t | grep Time | awk '{print $3}' >>/tmp/$t.time
+  done;
+  awk "{ sum += \$1 } END {printf \"%s%f # %s\n\", \"$c\", sum/50, \"$t\"}" /tmp/$t.time
+  c=","
+done
+
+echo "-- Run the following command to generate the new images:"
+echo "   R CMD BATCH images/graphs.R"

images/graphs.R

@@ -4,7 +4,7 @@ names.size <- c("thrift-binary", "thrift-compact", "protobuf", "boost", "msgpack
 names.time <- c("thrift-binary", "thrift-compact", "protobuf", "boost", "msgpack", "cereal", "avro", "yas", "yas-compact")
 names.time2 <- c("capnproto", "flatbuffers")
 # data from the 1000000 simulations
-# for t in thrift-binary thrift-compact protobuf boost msgpack cereal avro yas yas-compact; do echo -n "$t: "; ./benchmark 1 $t | grep size | awk '{print $4}'; done
+# c=" "; for t in thrift-binary thrift-compact protobuf boost msgpack cereal avro capnproto flatbuffers yas yas-compact; do ${BENCHMARK} -i 1 -s $t | grep Size | awk "{printf \"%s%d # %s\n\", \"$c\", \$3, \"$t\"}"; c=","; done
 size <- c(
      17017 # thrift-binary
     ,13378 # thrift-compact
@@ -18,21 +18,22 @@ size <- c(
     ,17416 # yas
     ,13321 # yas-compact
 )
-# for t in thrift-binary thrift-compact protobuf boost msgpack cereal avro yas yas-compact; do rm -f /tmp/$t.time; echo -n "$t: "; for i in `seq 1 50`; do ./benchmark 1000000 $t | grep time | awk '{print $4}' >>/tmp/$t.time; done; awk '{ sum += $1 } END { print sum/50}' /tmp/$t.time; done
+# c=" "; for t in thrift-binary thrift-compact protobuf boost msgpack cereal avro yas yas-compact; do rm -f /tmp/$t.time; for i in `seq 1 ${RESTARTS}`; do ${BENCHMARK} -i ${ITERATIONS} -s $t | grep Time | awk '{print $3}' >>/tmp/$t.time; done; awk "{ sum += \$1 } END {printf \"%s%f # %s\n\", \"$c\", sum/50, \"$t\"}" /tmp/$t.time; c=","; done
 time <- c(
-     1190.22 # thrift-binary
-    ,3474.32 # thrift-compact
-    ,2312.78 # protobuf
-    ,1195.04 # boost
-    ,2560.6  # msgpack
-    ,1052.46 # cereal
-    ,4488.18 # avro
-    ,302.7   # yas
-    ,2063.34 # yas-compact
+     13795.100000 # thrift-binary
+    ,41831.280000 # thrift-compact
+    ,30041.420000 # protobuf
+    ,14231.320000 # boost
+    ,31668.700000 # msgpack
+    ,14203.780000 # cereal
+    ,55352.740000 # avro
+    ,3307.360000 # yas
+    ,28981.900000 # yas-compact
 )
+# c=" "; for t in capnproto flatbuffers; do rm -f /tmp/$t.time; for i in `seq 1 ${RESTARTS}`; do ${BENCHMARK} -i ${ITERATIONS} -s $t | grep Time | awk '{print $3}' >>/tmp/$t.time; done; awk "{ sum += \$1 } END {printf \"%s%f # %s\n\", \"$c\", sum/50, \"$t\"}" /tmp/$t.time; c=","; done
 time2 <- c(
-     400.98 # capnproto
-    ,491.5  # flatbuffers
+     4512.320000 # capnproto
+    ,5821.920000 # flatbuffers
 )
 
 data.size <- as.data.frame(list(serializer = names.size, size = size))