From 3665806227d4fa948cc71f4d98775830dcc702b2 Mon Sep 17 00:00:00 2001 From: Mitchell Wheeler Date: Wed, 1 Jun 2022 10:14:25 +1000 Subject: [PATCH] Re-wrote the dockerfile to be a lot simpler (based on official osgeo/gdal images, which means we don't need to build anything ourselves) & should be less prone to things going out-of-date (osgeo will keep image to latest stable versions, any breakages we encounter would be us not supporting newer versions of GDAL = our problem) --- Dockerfile | 129 +++++++++++------------------------------------ requirements.txt | 2 +- 2 files changed, 31 insertions(+), 100 deletions(-) diff --git a/Dockerfile b/Dockerfile index ef2b7c8d8..f66b6478d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,110 +1,41 @@ -FROM ubuntu:16.04 +# Build: docker build -t pyrate . +FROM osgeo/gdal:ubuntu-small-3.4.3 -ENV LANG C.UTF-8 -ENV LANG=en_AU.UTF-8 -ENV WORKON_HOME=$HOME/venvs -ENV GDALINST=$HOME/gdalinstall -ENV GDALBUILD=$HOME/gdalbuild -ENV PROJINST=$HOME/gdalinstall -ENV GDALVERSION="3.0.2" -ENV PROJVERSION="6.1.1" -ENV PROJOPT="--with-proj=$GDALINST/gdal-$GDALVERSION" -ENV PATH=$GDALINST/gdal-$GDALVERSION/bin:$PATH -ENV LD_LIBRARY_PATH=$GDALINST/gdal-$GDALVERSION/lib:$LD_LIBRARY_PATH -ENV PROJ_LIB=$GDALINST/gdal-$GDALVERSION/share/proj -ENV GDAL_DATA=$GDALINST/gdal-$GDALVERSION/share/gdal - - -RUN apt-get update \ - && apt-get install -y build-essential checkinstall libreadline-gplv2-dev \ - libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev \ - libbz2-dev openssl curl libffi-dev - -RUN mkdir -p $HOME/opt - -RUN cd $HOME/opt \ - && curl -O https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz \ - && tar -xzf Python-3.7.7.tgz \ - && cd Python-3.7.7 \ - && ./configure --enable-shared --enable-optimizations --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" \ - && make altinstall +SHELL ["/bin/bash", "-c"] -RUN apt-get install -y build-essential python3-pip \ - apt-utils git libgdal-dev libatlas-base-dev openmpi-bin \ - libopenmpi-dev gfortran wget libhdf5-serial-dev sqlite3 vim +# update system and install pthon tooling + thirdparty deps +RUN apt-get update +RUN apt-get install -y python3-pip python3-venv python3-wheel python3-dev +RUN apt-get install -y openmpi-bin libopenmpi-dev sqlite3 -# update pip -RUN python3.7 -m pip install pip --upgrade -RUN python3.7 -m pip install wheel +# update setuptools RUN pip3 install --upgrade setuptools +# Copy PyRate source into image +COPY . /usr/src/PyRate +WORKDIR /usr/src/PyRate -RUN mkdir -p $GDALBUILD $GDALINST $PROJBUILD $PROJINST +# This is a hack the CI scripst also use, unit tests see, to assume it's read-only? +RUN chmod 444 tests/test_data/small_test/tif/geo_070709-070813_unw.tif -ENV GDALOPTS=" --with-geos \ - --with-expat \ - --without-libtool \ - --with-libz=internal \ - --with-libtiff=internal \ - --with-geotiff=internal \ - --without-gif \ - --without-pg \ - --without-grass \ - --without-libgrass \ - --without-cfitsio \ - --without-pcraster \ - --with-netcdf \ - --with-png=internal \ - --with-jpeg=internal \ - --without-gif \ - --without-ogdi \ - --without-fme \ - --without-hdf4 \ - --with-hdf5 \ - --without-jasper \ - --without-ecw \ - --without-kakadu \ - --without-mrsid \ - --without-jp2mrsid \ - --without-mysql \ - --without-ingres \ - --without-xerces \ - --without-odbc \ - --with-curl \ - --without-sqlite3 \ - --without-idb \ - --without-sde \ - --without-perl \ - --without-python" +# Create virtual env & install PyRate into it... +RUN python3 -m venv /usr/local/PyRate -RUN cd $PROJBUILD && wget -q https://download.osgeo.org/proj/proj-$PROJVERSION.tar.gz \ - && tar -xzf proj-$PROJVERSION.tar.gz \ - && cd proj-$PROJVERSION \ - && ./configure --prefix=$PROJINST/gdal-$GDALVERSION \ - && make -s -j 2 \ - && make install +# Dance around numpy/GDAL installation ordering requirements first... +RUN source /usr/local/PyRate/bin/activate && pip3 install $(grep numpy requirements.txt) +RUN source /usr/local/PyRate/bin/activate && pip3 install GDAL==$(gdal-config --version) -RUN cd $GDALBUILD \ - && wget -q http://download.osgeo.org/gdal/$GDALVERSION/gdal-$GDALVERSION.tar.gz \ - && tar -xzf gdal-$GDALVERSION.tar.gz +# Then install our requirements verbatim +RUN source /usr/local/PyRate/bin/activate && pip3 install -r requirements.txt +RUN source /usr/local/PyRate/bin/activate && pip3 install -r requirements-test.txt +RUN source /usr/local/PyRate/bin/activate && pip3 install -r requirements-dev.txt -RUN cd $GDALBUILD/gdal-$GDALVERSION \ - && ./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT +# For running like a program +RUN python3 setup.py install --prefix=/usr/local/PyRate +ENTRYPOINT [ "/bin/bash", "--init-file", "/usr/local/PyRate/bin/activate", "-i", "-c" ] -RUN cd $GDALBUILD/gdal-$GDALVERSION && make - -RUN cd $GDALBUILD/gdal-$GDALVERSION && make install - -RUN pip install virtualenv virtualenvwrapper -ENV VIRTUALENVWRAPPER_PYTHON=/usr/local/bin/python3.7 - -ADD . / PyRate/ -SHELL ["/bin/bash", "-c"] -RUN source /usr/local/bin/virtualenvwrapper.sh \ - && mkvirtualenv -p python3.7 pyrate \ - && cd PyRate \ - && sed -i '/^GDAL/d' requirements.txt \ - && workon pyrate \ - && pip install -r requirements.txt -r requirements-dev.txt -r requirements-test.txt \ - && pip install GDAL==$(gdal-config --version) \ - && python setup.py install +# For unit testing +ENV PYTHONPATH "/usr/src/PyRate:${PYTHONPATH}" +ENV OMPI_ALLOW_RUN_AS_ROOT 1 +ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM 1 +CMD [ "pytest", "-m", "'not slow and not mpi'" ] diff --git a/requirements.txt b/requirements.txt index e8629daa9..116796e39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -GDAL joblib==1.0.0 mpi4py==3.0.3 networkx==2.5 @@ -7,3 +6,4 @@ pyproj==3.0.0 scipy==1.5.4 numexpr==2.7.2 nptyping==1.4.0 +GDAL