From 3fb60670475ef10b6d07a4f761188b868ad278a2 Mon Sep 17 00:00:00 2001 From: "Mohar, Boaz" Date: Mon, 6 Jun 2016 22:54:26 -0400 Subject: [PATCH 1/2] BotoParallelReader bug not return 3 params as expected by accessor function --- thunder/readers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thunder/readers.py b/thunder/readers.py index 8034c7ab..50317d3a 100644 --- a/thunder/readers.py +++ b/thunder/readers.py @@ -391,7 +391,7 @@ def getsplit(kvIter): idx, keyName = kv key = bucket.get_key(keyName) buf = key.get_contents_as_string() - yield idx, buf + yield idx, buf, keyName npartitions = min(npartitions, self.nfiles) if npartitions else self.nfiles rdd = self.engine.parallelize(enumerate(keylist), npartitions) @@ -412,7 +412,7 @@ def getsplit(kv): idx, keyName = kv key = bucket.get_key(keyName) buf = key.get_contents_as_string() - return idx, buf + return idx, buf, keyName return [getsplit(kv) for kv in enumerate(keylist)] From 8c3d9b79b2e0dc62e833d3722328644ac7e22cc4 Mon Sep 17 00:00:00 2001 From: "Mohar, Boaz" Date: Mon, 6 Jun 2016 23:17:34 -0400 Subject: [PATCH 2/2] changed keyName to keyname --- thunder/readers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/thunder/readers.py b/thunder/readers.py index 50317d3a..b1e0f33a 100644 --- a/thunder/readers.py +++ b/thunder/readers.py @@ -388,10 +388,10 @@ def getsplit(kvIter): raise NotImplementedError("No file reader implementation for URL scheme " + scheme) for kv in kvIter: - idx, keyName = kv - key = bucket.get_key(keyName) + idx, keyname = kv + key = bucket.get_key(keyname) buf = key.get_contents_as_string() - yield idx, buf, keyName + yield idx, buf, keyname npartitions = min(npartitions, self.nfiles) if npartitions else self.nfiles rdd = self.engine.parallelize(enumerate(keylist), npartitions)