[WIP] Add update.py semi-automatic release crawler

.circleci/config.yml

@@ -0,0 +1,82 @@
+tag_filter: &tag_filter
+  filters:
+    tags:
+      only: /.*/
+    branches:
+      ignore: /.*/
+
+version: 2
+jobs:
+  build:
+    machine:
+      image: circleci/classic:latest
+    steps:
+      - checkout
+      - run:
+          name: Build Docker image
+          command: |
+            git describe --tags --always > version
+            docker build -t octomike/${CIRCLE_PROJECT_REPONAME,,} .
+            mkdir -p ${HOME}/docker
+            docker save "octomike/${CIRCLE_PROJECT_REPONAME,,}" > ~/docker/image.tar
+            # persist guessed branch so we can use it in deploy/tag
+            BRANCH=$(git branch --contains tags/${CIRCLE_TAG})
+            echo -n ${BRANCH} > ~/docker/branch
+          no_output_timeout: 30m # MCR is a large download
+      - persist_to_workspace:
+          root: /home/circleci
+          paths:
+            - docker/image.tar
+  test:
+    machine:
+      image: circleci/classic:latest
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Test Docker image
+          command: |
+            docker load -i /tmp/workspace/docker/image.tar
+            # figure out a better test
+            docker run -ti --rm --read-only --entrypoint /bin/sh octomike/${CIRCLE_PROJECT_REPONAME,,} -c 'test -d ${MCR_HOME}/runtime/glnxa64'
+  deploy:
+    docker:
+      - image: circleci/buildpack-deps:stretch
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - setup_remote_docker
+      - run: docker load -i /tmp/workspace/docker/image.tar
+      - run:
+          name: Publish Docker image
+          command: |
+            if [[ -n "${CIRCLE_TAG}" ]]; then
+              echo "${DOCKER_PASS}" | docker login --username "${DOCKER_USER}" --password-stdin
+              # tag should always be X.Y.Z[-variant]
+              docker tag octomike/${CIRCLE_PROJECT_REPONAME,,} octomike/${CIRCLE_PROJECT_REPONAME,,}:${CIRCLE_TAG}
+              docker push octomike/${CIRCLE_PROJECT_REPONAME,,}:${CIRCLE_TAG}
+              # also publish tag for the corresponding matlab release version, which is the name of the current branch
+              docker tag octomike/${CIRCLE_PROJECT_REPONAME,,} octomike/${CIRCLE_PROJECT_REPONAME,,}:${BRANCH}
+              docker push octomike/${CIRCLE_PROJECT_REPONAME,,}:${BRANCH}
+              BRANCH=$(cat /tmp/workspace/docker/branch)
+              # update major tag X.Y[-variant] to the latest in this branch
+              MAJOR_TAG=$(echo "${CIRCLE_TAG}" | sed -rn 's#([[:digit:]]+).([[:digit:]]+).([[:digit:]]+)(.*)#\1.\2\4#p')
+              if [[ -n "${MAJOR_TAG}" ]] ; then
+                docker tag octomike/${CIRCLE_PROJECT_REPONAME,,} octomike/${CIRCLE_PROJECT_REPONAME,,}:${MAJOR_TAG}
+                docker push octomike/${CIRCLE_PROJECT_REPONAME,,}:${MAJOR_TAG}
+              fi
+            fi
+workflows:
+  version: 2
+  build-test-deploy:
+    jobs:
+      - build:
+          <<: *tag_filter
+      - test:
+          requires:
+            - build
+          <<: *tag_filter
+      - deploy:
+          requires:
+            - test
+          <<: *tag_filter

Dockerfile-core.template

@@ -0,0 +1,42 @@
+FROM bids/base_validator
+
+# Update system
+RUN apt-get -qq update && apt-get -qq install -y \
+    unzip \
+    xorg \
+    wget && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install MATLAB MCR
+ENV MATLAB_VERSION %%MATLAB_VERSION%%
+RUN mkdir /opt/mcr_install && \
+    mkdir /opt/mcr && \
+    wget --quiet -P /opt/mcr_install %%MCR_LINK%% && \
+    unzip -q /opt/mcr_install/*${MATLAB_VERSION}*.zip -d /opt/mcr_install && \
+    cd /opt/mcr_install && mkdir save && \
+    for f in $(grep -E '(xml|enc)$' productdata/1000.txt) ; do  cp --parents archives/$f save/ ; done && \
+    for f in $(grep -E '(xml|enc)$' productdata/35000.txt) ; do  cp --parents archives/$f save/ ; done && \
+    for f in $(grep -E '(xml|enc)$' productdata/35010.txt) ; do  cp --parents archives/$f save/ ; done && \
+    rm -rf archives && mv save/archives . && rmdir save && \
+    /opt/mcr_install/install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent && \
+    rm -rf /opt/mcr_install /tmp/* && \
+    rm -rf /opt/mcr/*/cefclient && \
+    rm -rf /opt/mcr/*/mcr/toolbox/matlab/maps && \
+    rm -rf /opt/mcr/*/java/jarext && \
+    rm -rf /opt/mcr/*/toolbox/matlab/system/editor && \
+    rm -rf /opt/mcr/*/toolbox/matlab/codetools && \
+    rm -rf /opt/mcr/*/toolbox/matlab/datatools && \
+    rm -rf /opt/mcr/*/toolbox/matlab/codeanalysis && \
+    rm -rf /opt/mcr/*/toolbox/shared/dastudio && \
+    rm -rf /opt/mcr/*/toolbox/shared/mlreportgen && \
+    rm -rf /opt/mcr/*/sys/java/jre/glnxa64/jre/lib/ext/jfxrt.jar && \
+    rm -rf /opt/mcr/*/sys/java/jre/glnxa64/jre/lib/amd64/libjfxwebkit.so && \
+    rm -rf /opt/mcr/*/bin/glnxa64/libQt* && \
+    rm -rf /opt/mcr/*/bin/glnxa64/qtwebengine && \
+    rm -rf /opt/mcr/*/bin/glnxa64/cef_resources
+
+# Configure environment
+ENV MCR_VERSION %%MCR_VERSION%%
+ENV LD_LIBRARY_PATH /opt/mcr/${MCR_VERSION}/runtime/glnxa64:/opt/mcr/${MCR_VERSION}/bin/glnxa64:/opt/mcr/${MCR_VERSION}/sys/os/glnxa64:/opt/mcr/${MCR_VERSION}/sys/opengl/lib/glnxa64
+ENV MCR_INHIBIT_CTF_LOCK 1
+ENV MCR_HOME /opt/mcr/${MCR_VERSION}

Dockerfile → Dockerfile-full.template

@@ -8,16 +8,16 @@ RUN apt-get -qq update && apt-get -qq install -y \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install MATLAB MCR
-ENV MATLAB_VERSION R2016b
+ENV MATLAB_VERSION %%MATLAB_VERSION%%
 RUN mkdir /opt/mcr_install && \
     mkdir /opt/mcr && \
-    wget --quiet -P /opt/mcr_install http://www.mathworks.com/supportfiles/downloads/${MATLAB_VERSION}/deployment_files/${MATLAB_VERSION}/installers/glnxa64/MCR_${MATLAB_VERSION}_glnxa64_installer.zip && \
-    unzip -q /opt/mcr_install/MCR_${MATLAB_VERSION}_glnxa64_installer.zip -d /opt/mcr_install && \
+    wget --quiet -P /opt/mcr_install %%MCR_LINK%% && \
+    unzip -q /opt/mcr_install/*.zip -d /opt/mcr_install && \
     /opt/mcr_install/install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent && \
     rm -rf /opt/mcr_install /tmp/*
 
 # Configure environment
-ENV MCR_VERSION v91
+ENV MCR_VERSION %%MCR_VERSION%%
 ENV LD_LIBRARY_PATH /opt/mcr/${MCR_VERSION}/runtime/glnxa64:/opt/mcr/${MCR_VERSION}/bin/glnxa64:/opt/mcr/${MCR_VERSION}/sys/os/glnxa64:/opt/mcr/${MCR_VERSION}/sys/opengl/lib/glnxa64
 ENV MCR_INHIBIT_CTF_LOCK 1
-ENV MCRPath /opt/mcr/${MCR_VERSION}
+ENV MCR_HOME /opt/mcr/${MCR_VERSION}

update.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Dirty website parser to generate Dockerfiles for Matlab's MCR releases.
+
+Each Matlab release name (R2020a etc) gets a branch that contains a single
+Dockerfile and each release version (9.8.0 or 9.8.5 for 9.8 Update 5) becomes
+a tag in that branch. Each variant gets a new commit as well.
+So a sample history for R2020a could look something like this:
+
+ + [R2020a] <9.8.1-core> Auto-Update
+ + <9.8.1> Auto-Update
+ + Merged master
+ | \
+ |  + Update in templates
+ | /
+ + <9.8.0-core> Auto-Update
+ + <9.8.0> Auto-Update
+ + <master> Import
+
+(Circle)CI should then fire a docker build and push for every tag only. Shared
+tags for the major version (e.g. 9.8 always pointing to the latest 9.8 tag are
+done in Circle CI to avoid duplicate builds).
+
+Update.py should be run often enough to catch individual Matlab release updates.
+"""
+
+import re
+from subprocess import DEVNULL, run
+from urllib import request
+
+from packaging import version
+from bs4 import BeautifulSoup
+
+REL_URL = 'https://www.mathworks.com/products/compiler/matlab-runtime.html'
+VER_LIMIT = '9.3' # release URLs get weird before that..
+
+def call(cmd, split=True):
+    if split:
+        cmd = cmd.split()
+    process = run(cmd, stdout=DEVNULL, stderr=DEVNULL)
+    return process.returncode == 0
+
+
+with request.urlopen(REL_URL) as res:
+    if res.status != 200:
+        raise RuntimeError('Could not open matlab release URL')
+    html = res.read()
+
+soup = BeautifulSoup(html, 'html.parser')
+ver_re = re.compile(r'(R2\d{3}.) \((\d\.\d)\)')
+rel_re = re.compile(r'Release/(\d+)/')
+
+dockers = []
+for row in soup.find_all('table')[0].find_all('tr'):
+    tds = row.find_all('td')
+    if len(tds) >= 4:
+        name = tds[0].text
+        match = ver_re.match(name)
+        if not match:
+            continue
+        mcr_name, mcr_ver = match.groups()
+        if version.parse(mcr_ver) <= version.parse(VER_LIMIT):
+            continue
+        try:
+            link = tds[2].a.get('href')
+        except (KeyError, ValueError):
+            raise RuntimeError('Error parsing matlab release page')
+        if 'glnxa64' not in link:
+            raise RuntimeError('Error parsing matlab release page link')
+        match = rel_re.search(link)
+        if match:
+            mcr_ver = '{}.{}'.format(mcr_ver, match.groups()[0])
+        dockers.append((mcr_name, mcr_ver, link))
+
+
+variants = [
+    ('Dockerfile-full.template', ''),
+    ('Dockerfile-core.template', '-core')
+]
+new_tags = []
+
+for docker in dockers:
+    mcr_name, mcr_ver, link = docker
+    if len(mcr_ver.split('.')) == 2:
+        mcr_ver = mcr_ver + '.0'
+    mcr_ver_maj = '.'.join(mcr_ver.split('.')[0:2])
+    mcr_ver_dir = 'v{}'.format(mcr_ver_maj.replace('.', ''))
+    if not call('git checkout {}'.format(mcr_name)):
+        call('git checkout -b {}'.format(mcr_name))
+    for (template, suffix) in variants:
+        tag = '{}{}'.format(mcr_ver, suffix)
+        if call('git rev-parse --verify {}'.format(tag)):
+            print('Skipping {}/{}, already present'.format(mcr_name, tag))
+            continue
+        print('Adding {}/{}'.format(mcr_name, tag))
+        if not call('git merge master'):
+            raise RuntimeError('Merging master failed, will not continue')
+        with open(template) as f:
+            lines = f.read()
+            lines = lines.replace('%%MATLAB_VERSION%%', mcr_name)
+            lines = lines.replace('%%MCR_VERSION%%', mcr_ver_dir)
+            lines = lines.replace('%%MCR_LINK%%', link)
+            with open('Dockerfile', 'w+') as f2:
+                f2.write(lines)
+            call('git add Dockerfile')
+            # Tag X.Y.Z[-variant] - see circle CI for shared tag X.Y[-variant] 
+            call(['git', 'commit', '-m', 'Auto-Update'], split=False)
+            call('git tag {}'.format(tag))
+            new_tags.append(tag)
+    call('git checkout master')
+
+if new_tags:
+    print('New tags have been added, verify and update to git with:')
+    print('git push --all')
+    for tag in reversed(new_tags):
+        print('git push origin {}'.format(tag))