Merge pull request #701 from marscher/minor_improvements

Minor improvements:
* fix database corruption on OSX, safari in notebook sessions during kernel restart
* added FAQ about bitness on Windows
* fixed callbacks in progress reporter

doc/source/INSTALL.rst

@@ -21,10 +21,15 @@ but play at your own risk.
 If you already have a conda installation, directly go to step 3:
 
 1. Download and install miniconda for Python 2.7, 32 or 64 bit depending on your system:
-
    http://conda.pydata.org/miniconda.html
 
-   select **yes** to add conda to the **PATH** variable.
+
+   For Windows users, who do not know what to choose for 32 or 64 bit, it is strongly
+   recommended to read the second question of this FAQ first:
+   http://windows.microsoft.com/en-us/windows/32-bit-and-64-bit-windows
+
+
+   Run the installer and select **yes** to add conda to the **PATH** variable.
 
 2. If you have installed from a Linux shell, either open a new shell to have an updated PATH,
    or update your PATH variable by ``source ~/.bashrc`` (or .tcsh, .csh - whichever shell you are using).
@@ -206,4 +211,18 @@ Frequently Asked Questions (FAQ)
 
         conda config --add channels omnia
 
- 
+* Q: I'm using Windows, have a fresh Anaconda installation and I get strange errors
+  during "import xyz".
+
+  A: Possible answer 1: you have probably mixed 32 and 64 bit. Using 32 bit Python
+     on 64 bit Windows is fine, but not the other way around. 
+     Possible answer 2: Do you have Python2 and Python3 on the same computer?
+     To figure that you, open a cmd prompt and type in::
+
+     where python
+     "X:\somepath\miniconda2\Scripts\python.exe"
+
+     This should only display one line like. If it is displaying more than one .exe,
+     you either know what you are doing or you should remove one installation (eg. decide,
+     which branch of Python [2 or 3] to keep). 
+

pyemma/_base/progress/reporter.py

@@ -1,4 +1,3 @@
-
 # This file is part of PyEMMA.
 #
 # Copyright (c) 2015, 2014 Computational Molecular Biology Group, Freie Universitaet Berlin (GER)
@@ -15,17 +14,18 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 '''
 Created on 16.07.2015
 
 @author: marscher
 '''
+
 from __future__ import absolute_import, print_function
-from pyemma.util.types import is_int
+
 from pyemma._base.progress.bar import ProgressBar as _ProgressBar
 from pyemma._base.progress.bar import show_progressbar as _show_progressbar
 from pyemma._base.progress.bar.gui import hide_progressbar as _hide_progressbar
+from pyemma.util.types import is_int
 
 
 class ProgressReporter(object):
@@ -37,6 +37,30 @@ class ProgressReporter(object):
     # Note: this class has intentionally no constructor, because it is more
     # comfortable for the user of this class (who is then not in the need to call it).
 
+    @property
+    def show_progress(self):
+        if not hasattr(self, "_show_progress"):
+            self._show_progress = True
+        return self._show_progress
+
+    @show_progress.setter
+    def show_progress(self, val):
+        self._show_progress = bool(val)
+
+    @property
+    def _prog_rep_progressbars(self):
+        # stores progressbar representation per stage
+        if not hasattr(self, '_ProgressReporter__prog_rep_progressbars'):
+            self.__prog_rep_progressbars = {}
+        return self.__prog_rep_progressbars
+
+    @property
+    def _prog_rep_callbacks(self):
+        # store callback by stage
+        if not hasattr(self, '_ProgressReporter__callbacks'):
+            self.__callbacks = {}
+        return self.__callbacks
+
     def _progress_register(self, amount_of_work, description='', stage=0):
         """ Registers a progress which can be reported/displayed via a progress bar.
 
@@ -51,14 +75,9 @@ def _progress_register(self, amount_of_work, description='', stage=0):
             in the first pass over the data, calculate covariances in the second),
             one needs to estimate different times of arrival.
         """
-        if hasattr(self, 'show_progress') and not self.show_progress:
+        if not self.show_progress:
             return
 
-        # note this semantic makes it possible to use this class without calling
-        # its constructor.
-        if not hasattr(self, '_prog_rep_progressbars'):
-            self._prog_rep_progressbars = {}
-
         if not is_int(amount_of_work):
             raise ValueError("amount_of_work has to be of integer type. But is %s"
                              % type(amount_of_work))
@@ -71,12 +90,12 @@ class dummy(object):
             pg = dummy()
             pg.__str__ = lambda: description
             pg.__repr__ = pg.__str__
-            pg._dummy=None
+            pg._dummy = None
             pg.description = ''
         else:
             pg = _ProgressBar(amount_of_work, description=description)
-
         self._prog_rep_progressbars[stage] = pg
+        assert stage in self._prog_rep_progressbars
 
 #     def _progress_set_description(self, stage, description):
 #         """ set description of an already existing progress """
@@ -100,12 +119,9 @@ def register_progress_callback(self, call_back, stage=0):
         stage: int, optional, default=0
             The stage you want the given call back function to be fired.
         """
-        if hasattr(self, 'show_progress') and not self.show_progress:
+        if not self.show_progress:
             return
 
-        if not hasattr(self, '_callbacks'):
-            self._prog_rep_callbacks = {}
-
         assert callable(call_back)
         # check we have the desired function signature
         from pyemma.util.reflection import getargspec_no_self
@@ -116,6 +132,7 @@ def register_progress_callback(self, call_back, stage=0):
 
         if stage not in self._prog_rep_callbacks:
             self._prog_rep_callbacks[stage] = []
+
         self._prog_rep_callbacks[stage].append(call_back)
 
     def _progress_update(self, numerator_increment, stage=0):
@@ -129,12 +146,13 @@ def _progress_update(self, numerator_increment, stage=0):
             Current stage of the algorithm, 0 or greater
 
         """
-        if hasattr(self, 'show_progress') and not self.show_progress:
+        if not self.show_progress:
             return
 
         if stage not in self._prog_rep_progressbars:
             raise RuntimeError(
                 "call _progress_register(amount_of_work, stage=x) on this instance first!")
+
         if hasattr(self._prog_rep_progressbars[stage], '_dummy'):
             return
 
@@ -148,13 +166,13 @@ def _progress_update(self, numerator_increment, stage=0):
             raise Exception("This should not happen")
 
         _show_progressbar(pg)
-        if hasattr(self, '_prog_rep_callbacks'):
+        if hasattr(self, '_prog_rep_callbacks') and stage in self._prog_rep_callbacks:
             for callback in self._prog_rep_callbacks[stage]:
                 callback(stage, pg)
 
     def _progress_force_finish(self, stage=0):
         """ forcefully finish the progress for given stage """
-        if hasattr(self, 'show_progress') and not self.show_progress:
+        if not self.show_progress:
             return
         if stage not in self._prog_rep_progressbars:
             raise RuntimeError(

pyemma/_base/tests/test_progress.py

@@ -78,24 +78,18 @@ def test_callback(self):
         self.has_been_called = 0
 
         def call_back(stage, progressbar, *args, **kw):
-            global has_been_called
             self.has_been_called += 1
             assert isinstance(stage, int)
             assert isinstance(progressbar, ProgressBar)
 
-        class Worker(ProgressReporter):
-
-            def work(self, count):
-                self._progress_register(
-                    count, description="hard working", stage=0)
-                for _ in range(count):
-                    self._progress_update(1, stage=0)
-
-        worker = Worker()
+        amount_of_work = 100
+        worker = ProgressReporter()
+        worker._progress_register(
+            amount_of_work, description="hard working", stage=0)
         worker.register_progress_callback(call_back, stage=0)
-        expected_calls = 100
-        worker.work(count=expected_calls)
-        self.assertEqual(self.has_been_called, expected_calls)
+        for _ in range(amount_of_work):
+            worker._progress_update(1, stage=0)
+        self.assertEqual(self.has_been_called, amount_of_work)
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

pyemma/coordinates/data/util/traj_info_cache.py

@@ -122,6 +122,7 @@ class TrajectoryInfoCache(object):
 
     """
     _instance = None
+    DB_VERSION = '1'
 
     @staticmethod
     def instance():
@@ -130,17 +131,43 @@ def instance():
             cfg_dir = conf_values['pyemma']['cfg_dir']
             filename = os.path.join(cfg_dir, "trajlen_cache")
             TrajectoryInfoCache._instance = TrajectoryInfoCache(filename)
+            import atexit
+
+            @atexit.register
+            def write_at_exit():
+                if hasattr(TrajectoryInfoCache._instance._database, 'sync'):
+                    TrajectoryInfoCache._instance._database.sync()
+
         return TrajectoryInfoCache._instance
 
     def __init__(self, database_filename=None):
         if database_filename is not None:
-            self._database = anydbm.open(database_filename, flag="c")
+            try:
+                self._database = anydbm.open(database_filename, flag="c")
+            except anydbm.error as e:
+                try:
+                    os.unlink(database_filename)
+                    self._database = anydbm.open(database_filename, flag="n")
+                    # persist file right now, since it was broken
+                    self._set_curr_db_version(TrajectoryInfoCache.DB_VERSION)
+                    self._database.sync()
+                except OSError:
+                    raise RuntimeError('corrupted database in "%s" could not be deleted'
+                                       % os.path.abspath(database_filename))
         else:
             self._database = {}
 
-        self._database['db_version'] = '1'
+        self._set_curr_db_version(TrajectoryInfoCache.DB_VERSION)
         self._write_protector = Semaphore()
 
+    @property
+    def current_db_version(self):
+        return self._current_db_version
+
+    def _set_curr_db_version(self, val):
+        self._database['db_version'] = val
+        self._current_db_version = val
+
     def __getitem__(self, filename_reader_tuple):
         filename, reader = filename_reader_tuple
         key = self._get_file_hash(filename)
@@ -155,6 +182,11 @@ def __getitem__(self, filename_reader_tuple):
             # store info in db
             result = self.__setitem__(filename, info)
 
+            # save forcefully now
+            if hasattr(self._database, 'sync'):
+                logger.debug("sync db after adding new entry")
+                self._database.sync()
+
         return info
 
     def __format_value(self, traj_info):

pyemma/coordinates/tests/test_traj_info_cache.py

@@ -201,5 +201,16 @@ def test_old_db_conversion(self):
             assert info.ndim == 1
             assert info.offsets == []
 
+    def test_corrupted_db(self):
+        with NamedTemporaryFile(mode='w', suffix='.dat', delete=False) as f:
+            f.write("makes no sense!!!!")
+            f.close()
+        name = f.name
+        db = TrajectoryInfoCache(name)
+
+        # ensure we can perform lookups on the broken db without exception.
+        r = api.source(xtcfiles[0], top=pdbfile)
+        db[xtcfiles[0], r]
+
 if __name__ == "__main__":
     unittest.main()