diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py
index aa388c14ea1e..7a9c9b5cc8b7 100644
--- a/python/mxnet/test_utils.py
+++ b/python/mxnet/test_utils.py
@@ -1623,20 +1623,22 @@ def same_array(array1, array2):
 def discard_stderr():
     """
     Discards error output of a routine if invoked as:
-
     with discard_stderr():
         ...
     """
-
-    try:
-        stderr_fileno = sys.stderr.fileno()
-        old_stderr = os.dup(stderr_fileno)
-        bit_bucket = open(os.devnull, 'w')
-        os.dup2(bit_bucket.fileno(), stderr_fileno)
-        yield
-    finally:
-        os.dup2(old_stderr, stderr_fileno)
-        bit_bucket.close()
+    with open(os.devnull, 'w') as bit_bucket:
+        try:
+            stderr_fileno = sys.stderr.fileno()
+            old_stderr = os.dup(stderr_fileno)
+            try:
+                os.dup2(bit_bucket.fileno(), stderr_fileno)
+                yield
+            finally:
+                os.dup2(old_stderr, stderr_fileno)
+        except AttributeError:
+            # On some systems is stderr not a file descriptor but actually a virtual pipeline
+            # that can not be copied
+            yield
 
 class DummyIter(mx.io.DataIter):
     """A dummy iterator that always returns the same batch of data