Docker development setup based on Devuan images

jaromil · jaromil · commit d92f19cc1874 · 2018-09-04T14:31:57.000+02:00
Also moved travis test build to use this dockerfile
includes helper scripts and documentation updates
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,29 @@
+notifications:
+  email: false
+
+git:
+  submodules:
+    false
+
+env:
+  - RELEASE_BRANCH="master"
+
+sudo: required
+
+language: ruby
+
+services:
+  - docker
+
+before_install:
+  - docker build -t dyne/decodeos:travis docker-sdk
+
+script:
+  - container=$(docker create dyne/decodeos:travis)
+  - docker start $container
+  - docker exec  $container dam-client -gen
+  - sleep 10
+  - docker exec  $container ps axf
+  - docker stop  $container
+  - docker rm    $container
+
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,10 @@
+from dyne/devuan:ascii
+maintainer parazyd "https://github.com/parazyd"
+
+run echo "deb-src http://deb.devuan.org/merged ascii main" > /etc/apt/sources.list
+run echo "deb http://deb.devuan.org/merged ascii main"    >> /etc/apt/sources.list
+run apt-get -qq update
+run apt-get -yy install zsh cgpt parted xz-utils qemu qemu-utils python-markdown ruby-ronn --no-install-recommends
+copy . .
+run git submodule update --init --recursive --checkout
+
diff --git a/README.md b/README.md
@@ -1,33 +1,42 @@
-# Operating System for DEcentralised Data Ecosystems
+# Operating System for Private and Anonymous Computation Clusters
 
 [![software by Dyne.org](https://www.dyne.org/wp-content/uploads/2015/12/software_by_dyne.png)](http://www.dyne.org)
 
 <div class="center">
 
-The DECODE operating system is designed to run on servers, embedded
-computers and virtual machines to automatically connect applications
-to a private and anonymous peer-to-peer network cluster.
+The DECODE operating system is a brand new GNU+Linux distribution
+designed to run on servers, embedded computers and virtual machines to
+automatically connect micro-services to a private and anonymous
+peer-to-peer network cluster.
 
 </div>
 
-![DECODE OS logo](https://decodeos.dyne.org/img/decodeos_logo-800px.jpg)
+<img src="https://decodeos.dyne.org/img/decodeos_logo-800px.jpg" class="pic" alt="DECODE OS logo">
 
-| Features                                   | Components                                                                     |
-|--------------------------------------------|--------------------------------------------------------------------------------|
-| Wide compatibility with industry standards | GNU + Linux minimal base                                                       |
-| Anonimity and privacy by design            | [Tor](https://torproject.org) hidden service family                            |
-| Very secure, restricted environment        | [grsec](https://github.com/minipli/linux-unofficial_grsec/wiki) community fork |
-| Customisable to run different applications | [Devuan](https://devuan.org) GNU+Linux SDK                                                           |
-| Pluggable consensus algorithm              | [Redis](https://redis.io) based consensus broker                                                   |
-| Read-only and authenticated system         | SquashFS + overlayfs + Btrfs                                                   |
-| Integrated updating mechanism              | [Roundshot](https://github.com/DECODEproject/roundshot) initramfs              |
-| Low power consumption, outdoor usage       | Ports to embedded ARM boards                                                   |
+| Features                                   | Components                                                                                                                                                                                                  |
+|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Wide compatibility with industry standards | GNU + Linux minimal base                                                                                                                                                                                    |
+| Anonimity and privacy by design            | [Tor](https://torproject.org) hidden service family                                                                                                                                                         |
+| Very secure, restricted environment        | [grsec](https://github.com/minipli/linux-unofficial_grsec/wiki) community fork                                                                                                                              |
+| Customisable to run different applications | [Devuan](https://devuan.org) GNU+Linux SDK                                                                                                                                                                  |
+| Pluggable consensus algorithm              | [Redis](https://redis.io) based consensus broker                                                                                                                                                            |
+| Read-only and authenticated system         | [SquashFS](http://tldp.org/HOWTO/SquashFS-HOWTO/whatis.html) + [overlayfs](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) + [Btrfs](https://btrfs.wiki.kernel.org/index.php/Main_Page) |
+| Integrated updating mechanism              | [Roundshot](https://github.com/DECODEproject/roundshot) initramfs                                                                                                                                           |
+| Low power consumption, outdoor usage       | Ports to embedded ARM boards                                                                                                                                                                                |
+| Extensible platform support                | Includes latest JDK, Golang, Python etc.                                                                                                                                                                    |
+| Minimal resource consumption               | Online with less than 64MB of RAM                                                                                                                                                                           |
 
-For stable releases see https://files.dyne.org/decode
+## For stable releases see <a href="https://files.dyne.org/decode">files.dyne.org/decode</a>
 
-For more information about the DECODE project see https://decodeproject.eu
+## For more information see <a href="https://decodeproject.eu">the DECODE project</a>
 
-## Usage
+In particular, the following publications:
+
+- <a href="https://decodeproject.eu/publications/privacy-design-strategies-decode-architecture">Privacy Design Strategies for the DECODE Architecture</a>
+- <a href="https://decodeproject.eu/publications/decode-os-first-release">Decode OS first release</a>
+- <a href="https://decodeproject.eu/publications/decode-os-software-development-kit">DECODE OS Software Development Kit</a> (soon to be superseeded by the upcoming Devuan's Developer Manual)
+
+## Usage instructions
 
 DECODE OS comes in a variety of flavors:
 
@@ -42,26 +51,33 @@ The default username is `decode` with password `decode`
 
 The default `root` password is `toor`.
 
-## Support
+## Get in touch!
 
 Developers of the Dyne.org foundation are available to support
 customisations and adaptations of this operating system for particular
 purposes in line with the foundation's goals.
 
+You are welcome to contact us:
+
+ - **#devuan-dev** on **freenode** IRC (public, logged IPs)
+ - **#dyne** on <a href="https://irc.dyne.org">irc.dyne.org</a> (public and private, no IPs logged)
+ - E-mail **info@dyne.org**
+
 This project is a work in progress proceeding along a clear roadmap
-agreed for the DECODE project. The DECODE OS stable release is planned
-for 1st quarter 2019.
+agreed for the DECODE project. The DECODE OS **stable release is planned
+for 1st quarter 2019**.
 
-![Horizon 2020](https://zenroom.dyne.org/img/ec_logo.png)
+<img alt="Horizon 2020" src="https://zenroom.dyne.org/img/ec_logo.png" class="pic">
 
-This project is receiving funding from the European Union’s Horizon
+This project is receiving funding from the **European Union’s Horizon
 2020 research and innovation programme under grant agreement
-nr. 732546 (DECODE).
+nr. 732546**.
 
-## Build
+## Build from source
 
 The following instructions illustrate how one can build DECODE OS from
-scratch, eventually adding software to it.
+scratch, eventually adding software to it. This section is a work in
+progress.
 
 Building can be done from any GNU+Linux distribution, it entails
 bootstrapping a new Devuan base and then customising it via its SDK
@@ -73,7 +89,7 @@ Developers Manual", here is an outline on the steps to be taken.
 
 
 
-### Requirements
+### System requirements
 
 A GNU/Linux system is required in order to build DECODE OS.
 
diff --git a/docker-sdk/Dockerfile b/docker-sdk/Dockerfile
@@ -0,0 +1,122 @@
+#
+# Build this image with the command
+#   docker build -f docker/build -t dyne/clojure:latest
+#
+# Then run with the command
+#   docker run -p 3000:3000 -it dyne/clojure:latest
+#
+
+FROM dyne/devuan:beowulf
+ENV debian buster
+
+LABEL maintainer="Denis Roio <jaromil@dyne.org>" \
+	  homepage="https://github.com/decodeproject/decode-os"
+
+ENV LC_ALL C
+ENV DEBIAN_FRONTEND noninteractive
+
+# CLI arguments
+ARG foreground=true
+
+ENV DYNESDK=https://sdk.dyne.org:4443/job \
+	NETDATA_VERSION=1.10.0 \
+	STEM_VERSION=1.6.0 \
+	STEM_GIT=https://git.torproject.org/stem.git
+
+ENV BUILD_DEPS="build-essential zlib1g-dev gcc make autoconf automake pkg-config uuid-dev golang"
+WORKDIR /root
+
+# # debugging travis (finds gpg in local builds)
+RUN apt-get -yq update \
+&& apt-get -yq install gnupg1 ca-certificates --no-install-recommends \
+&& echo "ENVIRONMENT VARIABLES:" \
+&& export
+
+# Tor repository
+COPY tor.pub.asc tor.pub.asc
+RUN apt-key add tor.pub.asc
+RUN echo "deb https://deb.torproject.org/torproject.org $debian main" \
+	>> /etc/apt/sources.list
+
+# Nodejs repository
+ADD https://deb.nodesource.com/gpgkey/nodesource.gpg.key nodesource.gpg.key
+RUN apt-key add nodesource.gpg.key
+RUN	echo "deb https://deb.nodesource.com/node_8.x $debian main" \
+	>> /etc/apt/sources.list
+
+RUN mkdir -p /usr/share/man/man1/ \
+	&& apt-get -yy update && apt-get -yy upgrade \
+	&& apt-get -yy install tor deb.torproject.org-keyring \
+	   		   	   supervisor daemontools \
+	   		   	   tmux curl redis-tools redis-server net-tools \
+				   python3 python3-stem nodejs
+
+RUN apt-get -yq install $BUILD_DEPS
+
+# Latest Zenroom built static for x86-amd64 taken from our own builds at Dyne.org
+ADD $DYNESDK/zenroom-static-amd64/lastSuccessfulBuild/artifact/src/zenroom-static /usr/bin/zenroom
+RUN chmod +x /usr/bin/zenroom
+
+# Compile some software from the source
+WORKDIR /usr/src
+
+# Stem built from source
+# RUN git clone $STEM_GIT && cd stem && git checkout -b $STEM_VERSION $STEM_VERSION && python3 setup.py install
+
+# Configure Tor Controlport auth
+ENV	TORDAM_GIT=github.com/decodeproject/tor-dam
+RUN torpass=`echo "print(RNG.new():octet(16):base58())" | zenroom` \
+	&& go get -v -u $TORDAM_GIT/... && cd ~/go/src/github.com/decodeproject/tor-dam \
+	&& sed -i python/damhs.py -e "s/topkek/$torpass/" \
+	&& sed -i python/damauth.py -e "s/topkek/$torpass/" \
+	&& make install && make -C contrib install-init \
+	&& torpasshash=`HOME=/var/lib/tor setuidgid debian-tor tor --hash-password "$torpass"` \
+	&& sed -e 's/User tor/User debian-tor/' < contrib/torrc > /etc/tor/torrc \
+	&& sed -e 's/HashedControlPassword .*//' -i /etc/tor/torrc \
+	&& echo "HashedControlPassword $torpasshash" >> /etc/tor/torrc
+RUN chmod -R go-rwx /etc/tor && chown -R debian-tor /etc/tor \
+	&& rm -rf /var/lib/tor/data && chown -R debian-tor /var/lib/tor \
+	&& mkdir -p /var/run/tor && chown -R debian-tor /var/run/tor
+RUN cp /root/go/bin/dam* /usr/bin
+
+# fix npm - not the latest version installed by apt-get
+RUN npm install -g npm
+RUN npm install -g redis-commander
+ENV REDIS_HOSTS=localhost
+
+# Netdata
+ADD https://github.com/firehol/netdata/releases/download/v$NETDATA_VERSION/netdata-${NETDATA_VERSION}.tar.gz netdata.tgz
+RUN tar xf netdata.tgz && cd netdata-$NETDATA_VERSION \
+	&& ./netdata-installer.sh --dont-wait --dont-start-it \
+	&& cd - && rm -rf netdata.tgz netdata-$NETDATA_VERSION
+
+# Openresty
+ADD https://openresty.org/package/pubkey.gpg openresty.gpg
+RUN apt-key add openresty.gpg
+RUN echo "deb http://openresty.org/package/debian stretch openresty" \
+>> /etc/apt/sources.list
+RUN apt-get -yq update \
+&& apt-get -yq install --no-install-recommends openresty
+
+# cleanup
+RUN apt-get -yq remove --purge $BUILD_DEPS \
+&& apt-get -yq --purge autoremove && apt-get -yq clean \
+&& npm cache clean --force && npm uninstall -g npm
+
+COPY supervisord.conf /etc/supervisor/supervisord.conf
+RUN sed -i "s/nodaemon=true/nodaemon=$foreground/" /etc/supervisor/supervisord.conf
+
+RUN groupadd -g 6000 app && useradd -r -u 6000 -g app -d /home/app app
+WORKDIR /home/app
+RUN chown -R app:app /home/app
+
+# Tor's socks5
+EXPOSE 9150
+# supervisor
+EXPOSE 9001 9001
+# redis-commander
+EXPOSE 8081 8081
+# netdata
+EXPOSE 19999 19999
+
+CMD bash -c '/etc/init.d/supervisor start'
diff --git a/docker-sdk/README.md b/docker-sdk/README.md
@@ -0,0 +1,37 @@
+# DECODE OS - Docker facility
+
+
+[![Powered by DECODE OS](https://decodeos.dyne.org/img/decodeos_logo-800px.jpg)](https://decodeos.dyne.org)
+
+The DECODE operating system is a brand new GNU+Linux distribution designed to run on servers, embedded computers and virtual machines to automatically connect micro-services to a private and anonymous peer-to-peer network cluster.
+
+This is a Docker build of it to facilitate development and testing.
+
+## DO NOT USE IN PRODUCTION
+
+This Docker image is provided only for testing and showcase. We do release DECODE OS images for use in production on https://files.dyne.org/decode
+
+In order to test DECODE OS in Docker is possible to get the latest image with:
+```
+docker pull dyne/decodeos:latest
+```
+
+And then run it with:
+```
+docker run -it -p 9150 -p 9001:9001 -p 8081:8081 -p 19999:19999 dyne/decodeos:latest
+```
+
+Then connect to the web interfaces to monitor the functioning of DECODE OS:
+- http://localhost:9001 to supervise the daemons running and their logs
+- http://localhost:8081 to access the list of nodes and their values
+- http://localhost:19999 to monitor the resource usage
+
+At last, you can use localhost port 9150 using Socks5 connections to be routed through Tor. Your application may then interact with the listed nodes.
+
+## Build
+
+To re-build this docker image:
+```
+docker build dyne/decodeos:local .
+```
+
diff --git a/docker-sdk/exposed-ports b/docker-sdk/exposed-ports
@@ -0,0 +1,4 @@
+9001:9001
+8081:8081
+19999:19999
+9150:9150
diff --git a/docker-sdk/keygen b/docker-sdk/keygen
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+container=$(docker create dyne/decodeos:latest)
+container=$(docker start $container)
+onion=$(docker exec  $container dam-client -gen 2>&1| awk '/Our hostname/ {print $6}')
+echo "New DECODE-OS node address: $onion"
+image=$(docker commit $container dyne/decodeos:$onion)
+echo "Docker image: dyne/decodeos:$onion"
+echo "$image"
+container=$(docker stop $container)
+
+
diff --git a/docker-sdk/run b/docker-sdk/run
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+ports=""
+exposed=$(cat exposed-ports);
+for i in $exposed; do
+	ports="$ports -p $i"
+done
+echo "Starting DECODE OS Docker"
+for p in $exposed; do
+	echo "port exposed: http://localhost:`echo $p | cut -d: -f2`"
+done
+docker run -it $ports dyne/decodeos:latest
diff --git a/docker-sdk/supervisord.conf b/docker-sdk/supervisord.conf
@@ -0,0 +1,50 @@
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default $TEMP)
+nodaemon=true
+
+; supervisor web GUI
+[inet_http_server]
+port=*:9001
+; username=decode
+; password=decode
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=http://127.0.0.1:9001
+; username=decode
+; password=decode
+prompt=DECODE
+
+[program:tor]
+command=tor
+
+[program:dam-dir]
+command=dam-dir
+redirect_stderr=true
+
+[program:dam-client]
+command=dam-client
+redirect_stderr=true
+
+[program:redis-commander]
+command=redis-commander
+user=app
+
+[program:netdata]
+command=netdata -D
+user=netdata
+
+[group:network]
+programs=tor,dam-dir,dam-client,redis-commander,netdata
+priority=10
+umask=022
+autostart=true
+startsecs=10
diff --git a/docker-sdk/tor.pub.asc b/docker-sdk/tor.pub.asc
