Skip to content

Commit a13725b

Browse files
daveed12daveed12
authored
Merge pull request #1129 from grogsaxle/flatcar-arm64
Flatcar support
2 parents 7631424 + e260556 commit a13725b

File tree

2 files changed

+322
-0
lines changed

2 files changed

+322
-0
lines changed

‎pkg/flatcar.arm64/Dockerfile

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
# This Dockerfile aims to make building Hubble v4 packages easier.
2+
# Starting with version 4 building osquery is removed from individual Dockerfiles to its own.
3+
# osquery needs to be built once. Resulting tar file can be used in hubblev4 Dockerfiles.
4+
# Before building hubble, build osquery using a Dockerfile in pkg/osquery/ directory.
5+
# To build this image: 1. copy previously built osquery_4hubble.x86_64.tar to directory with this Dockerfile
6+
# 2. docker build -t <image_name> --build-arg HUBBLE_CHECKOUT=<tag or commit> --build-arg HUBBLE_VERSION=<Hubble Version> .
7+
# The resulting image is ready to build and run pyinstaller on container start that should
8+
# create hubble<version>-debian.tar.gz in the /data directory inside the container.
9+
# Mount /data volume into a directory on the host to access the package.
10+
# To run the container: docker run -it --rm -v `pwd`:/data <image_name>
11+
12+
FROM debian:9
13+
14+
RUN apt-get update \
15+
&& apt-get -y upgrade \
16+
&& apt-get clean \
17+
&& rm -rf /var/cache/apt
18+
19+
# paths that hubble or hubble parts need in the package
20+
RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
21+
22+
# install packages that should be needed for ligbit2 compilation and successful pyinstaller run
23+
RUN apt-get update \
24+
&& apt-get -y install git curl \
25+
python-dev libffi-dev libssl-dev libyaml-dev libssh2-1 libssh2-1-dev autoconf automake libtool \
26+
libxml2-dev libxslt1-dev python-cffi zlib1g-dev python-setuptools \
27+
cmake gcc wget python-pip openssl \
28+
&& apt-get clean \
29+
&& rm -rf /var/cache/apt
30+
31+
# libcurl install start
32+
# install libcurl to avoid depending on host version
33+
# requires autoconf libtool libssh2-devel zlib-devel autoconf
34+
ENV LIBCURL_SRC_URL=https://github.com/curl/curl.git
35+
ENV LIBCURL_SRC_VERSION=curl-7_64_1
36+
ENV LIBCURL_TEMP=/tmp/libcurl
37+
ENV PATH=/opt/hubble/bin/:/opt/hubble/include:/opt/hubble/lib:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
38+
RUN mkdir -p "$LIBCURL_TEMP" \
39+
&& cd "$LIBCURL_TEMP" \
40+
&& git clone "$LIBCURL_SRC_URL" \
41+
&& cd curl \
42+
&& git checkout "$LIBCURL_SRC_VERSION" \
43+
&& ./buildconf \
44+
&& ./configure --prefix=/opt/hubble --disable-ldap --without-nss --disable-manual --disable-gopher --disable-smtp --disable-smb --disable-imap --disable-pop3 --disable-tftp --disable-telnet --disable-dict --disable-ldaps --disable-ldap --disable-rtsp --with-libssh2 \
45+
&& make \
46+
&& make install \
47+
&& rm -rf "$LIBCURL_TEMP"
48+
49+
# git install start
50+
# install git so that git package won't be a package dependency
51+
# requires make git libcurl-devel autoconf zlib-devel gcc
52+
ENV GIT_SRC_URL=https://github.com/git/git.git
53+
ENV GIT_SRC_VERSION=v2.21.0
54+
ENV GITTEMP=/tmp/gittemp
55+
RUN mkdir -p "$GITTEMP" \
56+
&& cd "$GITTEMP" \
57+
&& git clone "$GIT_SRC_URL" \
58+
&& cd git \
59+
&& git checkout "$GIT_SRC_VERSION" \
60+
&& make configure \
61+
&& ./configure --prefix=/opt/hubble --with-tcltk=no --with-expat=no --with-python=no --with-curl=/opt/hubble \
62+
&& echo "NO_TCLTK=YesPlease" >> config.mak.autogen \
63+
&& echo "NO_PERL=YesPlease" >> config.mak.autogen \
64+
&& sed -i '0,/^NO_GETTEXT/s/^NO_GETTEXT.*/NO_GETTEXT=YesPlease/' config.mak.autogen \
65+
&& make \
66+
&& make install \
67+
&& rm -rf "$GITTEMP"
68+
69+
# clean up of /opt/hubble
70+
RUN rm /opt/hubble/bin/curl* \
71+
&& rm -rf /opt/hubble/include /opt/hubble/share
72+
73+
# libgit2 install start
74+
# must precede pyinstaller requirements
75+
ENV LIBGIT2_SRC_VERSION=1.1.0
76+
ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v${LIBGIT2_SRC_VERSION}.tar.gz
77+
# it turns out github provided release files can change. so even though the
78+
# code hopefully hasn't changed, the hash has.
79+
ENV LIBGIT2_SRC_SHA256=41a6d5d740fd608674c7db8685685f45535323e73e784062cf000a633d420d1e
80+
ENV LIBGIT2TEMP=/tmp/libgit2temp
81+
RUN mkdir -p "$LIBGIT2TEMP" \
82+
&& cd "$LIBGIT2TEMP" \
83+
&& wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
84+
&& echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \
85+
&& tar xzf libgit2.tar.gz \
86+
&& cd libgit2-"$LIBGIT2_SRC_VERSION"/ \
87+
&& export LIBGIT2=/usr/local/ \
88+
&& cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \
89+
&& make \
90+
&& make install \
91+
&& rm -rf "$LIBGIT2TEMP"
92+
93+
# fpm package making requirements start
94+
RUN apt-get install -y ruby ruby-dev rubygems gcc make \
95+
&& gem install --no-ri --no-rdoc ffi --version 1.12.2 \
96+
&& gem install --no-ri --no-rdoc fpm \
97+
&& apt-get clean \
98+
&& rm -rf /var/cache/apt
99+
100+
RUN apt-get install -y libbz2-dev libsqlite3-dev \
101+
&& apt-get clean \
102+
&& rm -rf /var/cache/apt
103+
104+
# use pyenv
105+
ARG PYENV_VERSION=3.7.9
106+
ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer
107+
ENV PYENV_ROOT=/opt/hubble/pyenv
108+
ENV PATH=$PYENV_ROOT/bin:$PATH
109+
ENV PYTHON_CONFIGURE_OPTS="--enable-shared"
110+
RUN umask 022 \
111+
&& curl -s -S -L "$PYENV_INSTALLER_URL" -o /usr/bin/pyenv-installer \
112+
&& chmod 0755 /usr/bin/pyenv-installer \
113+
&& /usr/bin/pyenv-installer \
114+
&& eval "$(pyenv init --path)" \
115+
&& pyenv install $PYENV_VERSION \
116+
&& pyenv global $PYENV_VERSION
117+
118+
RUN eval "$(pyenv init --path)" \
119+
&& pip -v install --upgrade pip
120+
121+
# pyinstaller start
122+
# commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
123+
# use the following variables to choose the version of hubble
124+
ARG HUBBLE_CHECKOUT=v4.5.0
125+
ARG HUBBLE_VERSION=4.5.0
126+
ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
127+
ENV HUBBLE_ITERATION=1
128+
ENV HUBBLE_URL=https://github.com/hubblestack/hubble
129+
ENV HUBBLE_SRC_PATH=/hubble_src
130+
ENV _HOOK_DIR="./pkg/"
131+
ENV _BINARY_LOG_LEVEL="INFO"
132+
ENV _INCLUDE_PATH=""
133+
ENV LD_LIBRARY_PATH=/opt/hubble/lib:/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64
134+
ENV HUBBLE_CHECKOUT=$HUBBLE_CHECKOUT
135+
ENV HUBBLE_VERSION=$HUBBLE_VERSION
136+
ENV HUBBLE_GIT_URL=$HUBBLE_GIT_URL
137+
138+
# leaving this blank will cause the entrypoint to look for either osquery_4hubble.tar
139+
# or osquery_4_hubble.$(uname -m).tar
140+
ARG OSQUERY_TAR_FILENAME=
141+
ENV OSQUERY_TAR_FILENAME=$OSQUERY_TAR_FILENAME
142+
143+
VOLUME /data
144+
WORKDIR /hubble_build
145+
COPY entrypoint.sh /entrypoint.sh
146+
ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]

‎pkg/flatcar.arm64/entrypoint.sh

+176
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
#!/bin/bash
2+
#Moving hubble source code logic in the shell script
3+
4+
# if ENTRYPOINT is given a CMD other than nothing
5+
# abort here and do that other CMD
6+
if [ $# -gt 0 ]
7+
then exec "$@"
8+
fi
9+
10+
set -x -e
11+
if [ ! -d "${HUBBLE_SRC_PATH}" ]
12+
then git clone "${HUBBLE_GIT_URL}" "${HUBBLE_SRC_PATH}"
13+
fi
14+
15+
if [ -n "$OSQUERY_TAR_FILENAME" ]
16+
then OSQUERY_TAR_FILENAMES=( /data/$OSQUERY_TAR_FILENAME )
17+
else OSQUERY_TAR_FILENAMES=( /data/osquery_4hubble.$(uname -m).tar /data/osquery_4hubble.tar )
18+
fi
19+
20+
if [ ! -d /opt/osquery ]
21+
then mkdir -vp /opt/osquery
22+
fi
23+
24+
for filename in "${OSQUERY_TAR_FILENAMES[@]}"; do
25+
if [ -e "$filename" ]; then
26+
# tar -C /opt/osquery -xvvf "$filename"
27+
tar -C / -xvvf "$filename"
28+
break
29+
fi
30+
done
31+
32+
ls -l /opt/osquery/bin
33+
34+
ln -s /opt/osquery/bin/osqueryd /opt/osquery/osqueryi
35+
ln -s /opt/osquery/bin/osqueryd /opt/osquery/osqueryd
36+
37+
if [ ! -L /opt/osquery/osqueryi ]
38+
then echo please provide a working osquery tarfile; exit 1
39+
else /opt/osquery/osqueryi --version
40+
fi
41+
42+
cd "${HUBBLE_SRC_PATH}"
43+
git checkout "${HUBBLE_CHECKOUT}"
44+
45+
HUBBLE_VERSION="$( sed -e 's/^v//' -e 's/[_-]rc/rc/g' <<< "$HUBBLE_VERSION" )"
46+
47+
cp -rf "${HUBBLE_SRC_PATH}"/* /hubble_build
48+
rm -rf /hubble_build/.git
49+
50+
cp /hubble_build/hubblestack/__init__.py /hubble_build/hubblestack/__init__.orig
51+
sed -i -e "s/BRANCH_NOT_SET/${HUBBLE_CHECKOUT}/g" -e "s/COMMIT_NOT_SET/$(cd ${HUBBLE_SRC_PATH}; git describe --long --always --tags)/g" /hubble_build/hubblestack/__init__.py
52+
cp /hubble_build/hubblestack/__init__.py /hubble_build/hubblestack/__init__.fixed
53+
54+
sed -i -e "s/'.*'/'$HUBBLE_VERSION'/g" /hubble_build/hubblestack/version.py
55+
56+
eval "$(pyenv init --path)"
57+
58+
# from now on, exit on error (rather than && every little thing)
59+
PS4=$'-------------=: '
60+
61+
# possibly replace the version file
62+
if [ -f /data/hubble_buildinfo ]; then
63+
echo >> /hubble_build/hubblestack/__init__.py
64+
cat /data/hubble_buildinfo >> /hubble_build/hubblestack/__init__.py
65+
fi 2>/dev/null
66+
67+
cat > /data/pre_packaged_certificates.py << EOF
68+
ca_crt = list()
69+
public_crt = list()
70+
EOF
71+
do_pkg_crts=0
72+
if [ -f /data/certs/ca-root.crt ]; then
73+
echo "ca_crt.append('''$(< /data/certs/ca-root.crt)''')" \
74+
>> /data/pre_packaged_certificates.py
75+
do_pkg_crts=$(( do_pkg_crts + 1 ))
76+
for item in /data/certs/int*.crt; do
77+
if [ -f "$item" ]
78+
then echo "ca_crt.append('''$(< "$item")''')" \
79+
>> /data/pre_packaged_certificates.py
80+
do_pkg_crts=$(( do_pkg_crts + 1 ))
81+
fi
82+
done
83+
fi
84+
for item in /data/certs/{pub,sign}*.crt; do
85+
if [ -f "$item" ]
86+
then echo "public_crt.append('''$(< "$item")''')" \
87+
>> /data/pre_packaged_certificates.py
88+
do_pkg_crts=$(( do_pkg_crts + 1 ))
89+
fi
90+
done
91+
if [ $do_pkg_crts -gt 0 ]
92+
then cp /data/pre_packaged_certificates.py /hubble_build/hubblestack
93+
fi
94+
95+
cd /hubble_build
96+
97+
# we may have preinstalled requirements that may need upgrading
98+
# pip install . might not upgrade/downgrade the requirements
99+
pip install wheel
100+
python setup.py egg_info
101+
pip install --upgrade \
102+
-r hubblestack.egg-info/requires.txt \
103+
-r optional-requirements.txt \
104+
-r package-requirements.txt
105+
pip freeze > /data/requirements.txt
106+
107+
[ -f ${_HOOK_DIR:-./pkg}/hook-hubblestack.py ] || exit 1
108+
109+
rm -rf build dist /opt/hubble/hubble-libs /hubble_build/hubble.spec
110+
export LD_LIBRARY_PATH=$(pyenv prefix)/lib:/opt/hubble/lib:/opt/hubble-libs
111+
export LD_RUN_PATH=$LD_LIBRARY_PATH
112+
pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL:-INFO} \
113+
--additional-hooks-dir ${_HOOK_DIR:-./pkg} \
114+
--runtime-hook pkg/runtime-hooks.py \
115+
./hubble.py 2>&1 | tee /tmp/pyinstaller.log
116+
117+
cp -pr dist/hubble /opt/hubble/hubble-libs
118+
119+
cat > /opt/hubble/hubble << EOF
120+
#!/bin/bash
121+
exec /opt/hubble/hubble-libs/hubble "\$@"
122+
exit 1
123+
EOF
124+
chmod 0755 /opt/hubble/hubble
125+
126+
[ -d /data/last-build.4 ] && rm -rf /data/last-build.4
127+
[ -d /data/last-build.3 ] && mv -v /data/last-build.3 /data/last-build.4
128+
[ -d /data/last-build.2 ] && mv -v /data/last-build.2 /data/last-build.3
129+
[ -d /data/last-build.1 ] && mv -v /data/last-build.1 /data/last-build.2
130+
cp -va build/ /data/last-build.1
131+
mv /tmp/pyinstaller.log /data/last-build.1
132+
cp -va /entrypoint.sh /data/last-build.1
133+
134+
mkdir -p /var/log/hubble_osquery/backuplogs
135+
136+
mkdir -p /etc/systemd/system
137+
mkdir -p /etc/profile.d
138+
mkdir -p /etc/hubble/hubble.d
139+
140+
cp -v /hubble_build/pkg/hubble.service /etc/hubble/hubble-example.service
141+
cp -v /hubble_build/conf/hubble-profile.sh /etc/profile.d/
142+
cp -v /hubble_build/conf/hubble-d-conf /etc/hubble/hubble.d
143+
144+
if [ -f /data/hubble ]
145+
then cp -v /data/hubble /etc/hubble/
146+
else cp -v /hubble_build/conf/hubble /etc/hubble/
147+
fi
148+
149+
if [ "X$NO_TAR" = X1 ]; then
150+
echo "exiting (as requested by NO_TAR=$NO_TAR) without pre-tar-ing package"
151+
exit 0
152+
fi 2>/dev/null
153+
154+
# also bring in anything from a /data/opt/ directory so we can bundle other executables if needed
155+
if [ -d /data/opt ]
156+
then cp -r /data/opt/* /opt
157+
fi
158+
159+
PACKAGE_NAME_ARCH="${ARCH:-$(uname -m)}"
160+
161+
# edit to change iteration number, if necessary
162+
PKG_BASE_NAME=hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}
163+
PKG_OUT_EXT=$PACKAGE_NAME_ARCH.tar.gz
164+
PKG_FIN_EXT=flatcar.$PKG_OUT_EXT
165+
PKG_FNAME=$PKG_BASE_NAME.$PKG_FIN_EXT
166+
PKG_FILE="/data/$PKG_FNAME"
167+
168+
tar -cSPvvzf "$PKG_FILE" \
169+
--exclude opt/hubble/pyenv \
170+
/etc/hubble /opt/hubble /opt/osquery \
171+
/etc/profile.d/hubble-profile.sh \
172+
/var/log/hubble_osquery/backuplogs \
173+
/etc/hubble/hubble-example.service \
174+
2>&1 | tee /hubble_build/deb-pkg-start-tar.log
175+
176+
openssl dgst -sha256 "$PKG_FILE" > "$PKG_FILE".sha256

0 commit comments

Comments
 (0)