Merge pull request #1512 from dwf/hantek_nan2

Rebase of #1486

Author: dwf

pylearn2/devtools/nan_guard.py

@@ -11,6 +11,8 @@
 import logging
 from theano.compile import Mode
 import theano
+import theano.tensor as T
+import theano.sandbox.cuda as cuda
 import numpy as np
 from pylearn2.models.dbm import flatten
 from pylearn2.utils import contains_nan, contains_inf
@@ -36,6 +38,23 @@ class NanGuardMode(Mode):
         If True, raise an error when a value greater than 1e10 is encountered.
     """
     def __init__(self, nan_is_error, inf_is_error, big_is_error=True):
+        if cuda.cuda_available:
+            self.guard_input = cuda.fvector('nan_guard')
+            if nan_is_error or inf_is_error:
+                self.gpumin = theano.function(
+                    [self.guard_input], T.min(self.guard_input),
+                    mode='FAST_RUN'
+                )
+            if inf_is_error:
+                self.gpumax = theano.function(
+                    [self.guard_input], T.max(self.guard_input),
+                    mode='FAST_RUN'
+                )
+            if big_is_error:
+                self.gpuabsmax = theano.function(
+                    [self.guard_input], T.max(T.abs_(self.guard_input)),
+                    mode='FAST_RUN'
+                )
         def do_check_on(var, nd, f, is_input):
             """
             Checks `var` for NaNs / Infs. If detected, raises an exception
@@ -56,15 +75,31 @@ def do_check_on(var, nd, f, is_input):
             """
             error = False
             if nan_is_error:
-                if contains_nan(var):
+                err = False
+                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):
+                    err = np.isnan(self.gpumin(var.reshape(var.size)))
+                else:
+                    err = contains_nan(var)
+                if err:
                     logger.error('NaN detected')
                     error = True
             if inf_is_error:
-                if contains_inf(var):
+                err = False
+                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):
+                    err = (np.isinf(self.gpumin(var.reshape(var.size))) or \
+                           np.isinf(self.gpumax(var.reshape(var.size))))
+                else:
+                    err = contains_inf(var)
+                if err:
                     logger.error('Inf detected')
                     error = True
             if big_is_error:
-                if np.abs(var).max() > 1e10:
+                err = False
+                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):
+                    err = (self.gpuabsmax(var.reshape(var.size)) > 1e10)
+                else:
+                    err = (np.abs(var).max() > 1e10)
+                if err:
                     logger.error('Big value detected')
                     error = True
             if error:

pylearn2/devtools/tests/test_nan_guard.py

@@ -0,0 +1,49 @@
+"""
+This test is for testing the NanGuardMode.
+"""
+from pylearn2.devtools.nan_guard import NanGuardMode
+import numpy
+import theano
+import theano.tensor as T
+
+
+def test_NanGuardMode():
+    """
+    Tests if NanGuardMode is working by feeding in numpy.inf and numpy.nans
+    intentionally. A working implementation should be able to capture all
+    the abnormalties.
+    """
+    x = T.matrix()
+    w = theano.shared(numpy.random.randn(5, 7).astype(theano.config.floatX))
+    y = T.dot(x, w)
+
+    fun = theano.function(
+        [x], y,
+        mode=NanGuardMode(nan_is_error=True, inf_is_error=True)
+    )
+    a = numpy.random.randn(3, 5).astype(theano.config.floatX)
+    infa = numpy.tile(
+        (numpy.asarray(100.) ** 1000000).astype(theano.config.floatX), (3, 5))
+    nana = numpy.tile(
+        numpy.asarray(numpy.nan).astype(theano.config.floatX), (3, 5))
+    biga = numpy.tile(
+        numpy.asarray(1e20).astype(theano.config.floatX), (3, 5))
+
+    work = [False, False, False]
+
+    fun(a)  # normal values
+    try:
+        fun(infa)  # INFs
+    except AssertionError:
+        work[0] = True
+    try:
+        fun(nana)  # NANs
+    except AssertionError:
+        work[1] = True
+    try:
+        fun(biga)  # big values
+    except AssertionError:
+        work[2] = True
+
+    if not (work[0] and work[1] and work[2]):
+        raise AssertionError("NanGuardMode not working.")