Merging devel into ref_thermo

.travis.yml

@@ -9,6 +9,10 @@ env:
   global:
     - PATH=$HOME/miniconda/bin:$PATH
     - common_py_deps="jinja2 conda-build"
+    - PACKAGENAME=pyemma
+    - ORGNAME=omnia
+    - DEV_BUILD_N_KEEP=10 
+    - secure: "YI89RGfpxB29XoyPzBGXsCWSgdhP4i+DS7gFmwaDoOz3R+ZW8yG3ZaexW15WUe7h0tb1L2aYvZCcqgaBrH2SNCiR0SRbhk3EBPg1C3baMVlVhDeQru9S/FrEJ1ZUGhnUitbEEIdG2MYexETjoGef6K+7dXBJWJMy/rNRm61PSJw="
   matrix:
     # minimum possible numpy version = 1.8, because of mdtraj
     - python=2.7  CONDA_PY=27  CONDA_NPY=19
@@ -25,16 +29,16 @@ matrix:
        env: python=2.7  CONDA_PY=27  CONDA_NPY=19
 
 before_install:
-- tools/ci/travis/install_miniconda.sh
+- devtools/ci/travis/install_miniconda.sh
 - conda config --set always_yes true 
 - conda config --add channels omnia 
 - conda install -q $common_py_deps 
 
 script:
-- conda build -q tools/conda-recipe
+- conda build -q devtools/conda-recipe
 
 after_success:
 # coverage report: needs .coverage file generated by testsuite and git src
 - pip install coveralls
 - coveralls 
-
+- if [ "$TRAVIS_SECURE_ENV_VARS" == true ]; then source devtools/ci/travis/after_success.sh; fi

appveyor.yml

@@ -1,28 +1,29 @@
 environment:
+  BINSTAR_TOKEN: o5nTvV9PwJQgt5HXoqAl1iWi1ivyOrGb9ZL20puP4xklAEfvN/0EDHbhDHpVwjC7
   global:
     # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
     # /E:ON and /V:ON options are not enabled in the batch script intepreter
     # See: http://stackoverflow.com/a/13751649/163740
-    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\tools\\ci\\appveyor\\run_with_env.cmd"
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\devtools\\ci\\appveyor\\run_with_env.cmd"
     CONDA_NPY: "110"
 
   matrix:
-    - PYTHON: "C:\\Miniconda"
+    - MINICONDA_PYTHON: "C:\\Miniconda"
       CONDA_PY: "27"
 
-    - PYTHON: "C:\\Miniconda-x64"
+    - MINICONDA_PYTHON: "C:\\Miniconda-x64"
       CONDA_PY: "27" 
       ARCH: "64"
 
-    - PYTHON: "C:\\Miniconda3"
+    - MINICONDA_PYTHON: "C:\\Miniconda3"
       CONDA_PY: "35"
 
-    - PYTHON: "C:\\Miniconda3-x64"
+    - MINICONDA_PYTHON: "C:\\Miniconda3-x64"
       CONDA_PY: "34"
       ARCH: "64"
 
 install:
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%;"
+  - "SET PATH=%MINICONDA_PYTHON%;%MINICONDA_PYTHON%\\Scripts;%PATH%;"
 
   - conda config --set always_yes true 
   - conda config --add channels omnia 
@@ -37,4 +38,8 @@ build: false # Not a C# project, build stuff at the test step instead.
 
 test_script:
   # run testsuite and upload test results to AppVeyor; return exit code of testsuite
-  - conda build -q tools/conda-recipe
+  - conda build -q devtools/conda-recipe
+
+#on_success:
+#  - devtools/ci/appveyor/after_success.bat  
+

devtools/ci/appveyor/after_success.bat

@@ -0,0 +1,7 @@
+conda install --yes -q anaconda-client jinja2
+cd %PYTHON_MINICONDA%\conda-bld
+dir /s /b %PACKAGENAME%-dev-*.tar.bz2 > files.txt
+for /F %%filename in (files.txt) do (
+    echo "uploading file %%~filename"
+    anaconda -t %BINSTAR_TOKEN% upload --force -u %ORGNAME% -p %PACKAGENAME%-dev %%~filename
+)

tools/ci/appveyor/runTestsuite.ps1 → devtools/ci/appveyor/runTestsuite.ps1

@@ -22,7 +22,7 @@ function upload($file) {
 
 function run {
     cd $env:APPVEYOR_BUILD_FOLDER
-    $stylesheet =  "tools/ci/appveyor/transform_xunit_to_appveyor.xsl"
+    $stylesheet =  "devtools/ci/appveyor/transform_xunit_to_appveyor.xsl"
     $input = "nosetests.xml"
     $output = "transformed.xml"
 

devtools/ci/travis/after_success.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+if [ "$TRAVIS_PULL_REQUEST" = true ]; then
+    echo "This is a pull request. No deployment will be done."; exit 0
+fi
+
+
+if [ "$TRAVIS_BRANCH" != "devel" ]; then
+    echo "No deployment on BRANCH='$TRAVIS_BRANCH'"; exit 0
+fi
+
+
+# Deploy to binstar
+conda install --yes anaconda-client jinja2
+pushd .
+cd $HOME/miniconda/conda-bld
+FILES=*/${PACKAGENAME}-dev-*.tar.bz2
+for filename in $FILES; do
+    anaconda -t $BINSTAR_TOKEN upload --force -u ${ORGNAME} -p ${PACKAGENAME}-dev ${filename}
+done
+popd
+
+# call cleanup
+python devtools/ci/travis/dev_pkgs_del_old.py

devtools/ci/travis/dev_pkgs_del_old.py

@@ -0,0 +1,45 @@
+"""
+Cleanup old development builds on Anaconda.org
+ 
+Assumes one has set two environment variables:
+
+1. BINSTAR_TOKEN: token to authenticate with anaconda.org
+2. DEV_BUILD_N_KEEP: int, how many builds to keep, delete oldest first.
+3. ORGNAME: str, anaconda.org organisation/user
+4. PACKGENAME: str, name of package to clean up
+
+author: Martin K. Scherer
+data: 20.4.16
+"""
+from __future__ import print_function, absolute_import
+from binstar_client.utils import get_server_api
+from pkg_resources import parse_version
+from operator import getitem
+import os
+
+token = os.getenv['BINSTAR_TOKEN']
+org = os.getenv['ORGNAME']
+pkg = os.getenv['PACKAGENAME']
+n_keep = int(os.getenv('DEV_BUILD_N_KEEP'))
+
+b = get_server_api(token=token)
+package = b.package(org, pkg)
+
+# sort releases by version number, oldest first
+sorted_by_version = sorted(package['releases'],
+                           key=lambda rel: parse_version(rel['version']),
+                           reverse=True
+                          )
+to_delete = []
+
+while len(sorted_by_version) > N_KEEP:
+    to_delete.append(sorted_by_version.pop())
+
+# remove old releases from anaconda.org 
+for rel in to_delete:
+    spec = rel['full_name']
+    version = rel['version']
+    for dist in rel['distributions']:
+        b.remove_dist(org, package_name=pkg, release=version, basename=dist)
+        print("removed file %s" % dist)
+

devtools/conda-recipe/bld.bat

@@ -0,0 +1,10 @@
+if not defined APPVEYOR (
+    echo not on appveyor
+    "%PYTHON%" setup.py install
+) else ( 
+    echo on appveyor
+    cmd /E:ON /V:ON /C %APPVEYOR_BUILD_FOLDER%\devtools\ci\appveyor\run_with_env.cmd "%PYTHON%" setup.py install
+)
+set build_status=%ERRORLEVEL%
+"%PYTHON%" devtools\conda-recipe\dev_version.py
+if %build_status% == 1 exit 1

devtools/conda-recipe/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+$PYTHON setup.py install
+$PYTHON devtools/conda-recipe/dev_version.py

devtools/conda-recipe/dev_version.py

@@ -0,0 +1,5 @@
+from __future__ import print_function
+from pyemma import __version__ as version
+
+with open('__conda_version__.txt', 'w') as f:
+    f.write(version)

tools/conda-recipe/meta.yaml → devtools/conda-recipe/meta.yaml

@@ -1,6 +1,6 @@
 package:
-  name: pyemma
-  version: !!str dev
+  name: pyemma-dev
+  #version: 0 
 source:
   path: ../..
 

doc/Makefile

@@ -1,6 +1,7 @@
 # Makefile for Sphinx documentation
 #
 
+SHELL := /bin/bash
 PYVER = 2.7
 PYTHON = python$(PYVER)
 
@@ -17,8 +18,8 @@ PAPEROPT_letter = -D latex_paper_size=letter
 ALLSPHINXOPTS   = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
 # ipython notebook index file
-F			 = source/generated/ipynb-list.rst
-HEADER = '.. toctree:: \n   :maxdepth: 1 \n'
+NBCONVERT_CONFIG = $(realpath jupyter_nbconvert_config.py)
+NBCONVERT_CMD = jupyter nbconvert $(nbflags) --config $(NBCONVERT_CONFIG)
 
 .PHONY: help clean html ipython-rst
 
@@ -42,20 +43,10 @@ source/generated:
 	mkdir -p source/generated
 
 ipython-rst: source/generated
-	# find all ipython notebooks (without their checkpoints)
-	@$(eval notebooks:= $$(shell cd source/generated && \
-                               find ../../../pyemma-ipython/ -name \*.ipynb \
-                                    -and -not -name \*-checkpoint.ipynb))
-	# convert notebooks to rst
-	cd source/generated; jupyter nbconvert --to rst $(notebooks)
-	# strip directory part and extensions
-	#@$(eval notebooks=$$(notdir $(notebooks)))
-	#@$(eval notebooks=$$(basename $(notebooks)))
-	# write header
-	#@echo $(HEADER) > $(F)
-	# put every notebook as a link in index file
-	#@$(foreach n,$(notebooks), echo "   $n" >> $(F);)
-
+	@$(eval nb_names:= $$(shell grep generated source/ipython.rst | sed -re 's/\s+generated\/(.+)/\/\1.ipynb|/g'))
+	@$(eval notebooks:= $$(shell find ../pyemma-ipython -name *.ipynb | grep -E "$(nb_names)" ))
+	cd source/generated
+	set -x; for nb in $(notebooks); do $(NBCONVERT_CMD) $$nb; done
 	@echo "Conversion finished."
 
 latex:
@@ -64,3 +55,4 @@ latex:
 
 html_check: 
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/html $(FILES)
+