From 4038f9887aeee615b2fa0dd1268953f528a33849 Mon Sep 17 00:00:00 2001 From: Neal Shail Date: Thu, 8 Mar 2018 19:27:33 +0000 Subject: [PATCH 1/5] debugging --- lib/index.js | 60 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/lib/index.js b/lib/index.js index 526f5f1..97669b6 100644 --- a/lib/index.js +++ b/lib/index.js @@ -427,8 +427,8 @@ Client.prototype.downloadFile = function(params) { var downloader = new EventEmitter(); var localFile = params.localFile; var s3Params = extend({}, params.s3Params); - - var dirPath = path.dirname(localFile); + var dirPath = (localFile.match(/\/$/))?localFile:path.dirname(localFile); + console.log('downloadFile', dirPath); downloader.progressAmount = 0; mkdirp(dirPath, function(err) { if (err) { @@ -1018,6 +1018,9 @@ function syncDir(self, params, directionIsToS3) { return; } + console.log('localpath=', (localFileStat)?localFileStat.path:'null', ' locals3path=', (localFileStat)?localFileStat.s3Path:'null', ' s3Object.key=', (s3Object)?s3Object.key:'null'); + + // special case for directories when deleteRemoved is true and we're // downloading a dir from S3. We don't add directories to the list // unless this case is true, so we assert that fact here. @@ -1025,12 +1028,42 @@ function syncDir(self, params, directionIsToS3) { assert.ok(!directionIsToS3); assert.ok(deleteRemoved); - localFileCursor += 1; - setImmediate(checkDoMoreWork); - - if (!s3Object || s3Object.key.indexOf(localFileStat.s3Path) !== 0) { + if (!s3Object) { + console.log('!!! !s3Object'); + deleteLocalDir(); // no more s3 objects + } + else if (localFileStat.s3Path > s3Object.key){ + downloadS3Object(); // previous item, download it. + } + else if (s3Object.key.indexOf(localFileStat.s3Path) !== 0 ){ // starts with this folder name + console.log('!!! s3Object.key.indexOf(localFileStat.s3Path) !== 0'); deleteLocalDir(); + } else if (localFileStat.s3Path === s3Object.key) { // same folder (don't need to download it.) + skipThisOne(); + } else if (s3Object.key.indexOf(localFileStat.s3Path) === 0){ + localFileCursor += 1; + setImmediate(checkDoMoreWork); // skip this, just a local folder } + + + // different cases + + // a directory, with no remote equivalent (has been deleted) + // local=/xml remote=undefined <-- at the end. ** + // local=/xml remote=/yml/head.hbs delete it + + // a directory with a remove object equivalent + // local=/xml remote=/xml + // --> ignore it. + + // a directory, with no remote object equivalent (but a file that begins with the same name) + // local=/xml remote=/xml/something.hbs + // --> do nothing just pass on.. + return; } @@ -1069,8 +1102,12 @@ function syncDir(self, params, directionIsToS3) { } function deleteLocalDir() { - var fullPath = path.join(localDir, localFileStat.path); + localFileCursor += 1; + setImmediate(checkDoMoreWork); + if (!deleteRemoved) return; ee.deleteTotal += 1; + var fullPath = path.join(localDir, localFileStat.path); + console.log('removing directory', fullPath); rimraf(fullPath, function(err) { if (fatalError) return; if (err && err.code !== 'ENOENT') return handleError(err); @@ -1086,6 +1123,7 @@ function syncDir(self, params, directionIsToS3) { if (!deleteRemoved) return; ee.deleteTotal += 1; var fullPath = path.join(localDir, localFileStat.path); + console.log('deleting:'+fullPath); fs.unlink(fullPath, function(err) { if (fatalError) return; if (err && err.code !== 'ENOENT') return handleError(err); @@ -1100,6 +1138,8 @@ function syncDir(self, params, directionIsToS3) { setImmediate(checkDoMoreWork); var fullPath = path.join(localDir, toNativeSep(s3Object.key)); + console.log('downloadS3Object', fullPath); + if (getS3Params) { getS3Params(fullPath, s3Object, haveS3Params); } else { @@ -1146,6 +1186,7 @@ function syncDir(self, params, directionIsToS3) { } function skipThisOne() { + console.log('skipThisOne localFile=' + allLocalFiles[localFileCursor].path + ' s3Object=' +allS3Objects[s3ObjectCursor].key); s3ObjectCursor += 1; localFileCursor += 1; setImmediate(checkDoMoreWork); @@ -1237,6 +1278,7 @@ function syncDir(self, params, directionIsToS3) { ee.emit('progress'); data.Contents.forEach(function(object) { object.key = object.Key.substring(prefix.length); + console.log('s3Object=', object.key); allS3Objects.push(object); }); checkDoMoreWork(); @@ -1266,6 +1308,10 @@ function syncDir(self, params, directionIsToS3) { return 0; } }); + + for (var i = 0; i < allLocalFiles.length; i++) { + console.log('localFile=' + allLocalFiles[i].path + ', s3Path=' + allLocalFiles[i].s3Path); + } startComputingMd5Sums(); }); } From c29d14e5015b5f56fe2dcf701dd80cf3b8b084d7 Mon Sep 17 00:00:00 2001 From: Neal Shail Date: Thu, 8 Mar 2018 19:47:34 +0000 Subject: [PATCH 2/5] cleaned up debug and comments --- lib/index.js | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/lib/index.js b/lib/index.js index 97669b6..f5c68a1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -428,7 +428,6 @@ Client.prototype.downloadFile = function(params) { var localFile = params.localFile; var s3Params = extend({}, params.s3Params); var dirPath = (localFile.match(/\/$/))?localFile:path.dirname(localFile); - console.log('downloadFile', dirPath); downloader.progressAmount = 0; mkdirp(dirPath, function(err) { if (err) { @@ -1018,8 +1017,6 @@ function syncDir(self, params, directionIsToS3) { return; } - console.log('localpath=', (localFileStat)?localFileStat.path:'null', ' locals3path=', (localFileStat)?localFileStat.s3Path:'null', ' s3Object.key=', (s3Object)?s3Object.key:'null'); - // special case for directories when deleteRemoved is true and we're // downloading a dir from S3. We don't add directories to the list @@ -1029,14 +1026,12 @@ function syncDir(self, params, directionIsToS3) { assert.ok(deleteRemoved); if (!s3Object) { - console.log('!!! !s3Object'); deleteLocalDir(); // no more s3 objects } else if (localFileStat.s3Path > s3Object.key){ downloadS3Object(); // previous item, download it. } else if (s3Object.key.indexOf(localFileStat.s3Path) !== 0 ){ // starts with this folder name - console.log('!!! s3Object.key.indexOf(localFileStat.s3Path) !== 0'); deleteLocalDir(); } else if (localFileStat.s3Path === s3Object.key) { // same folder (don't need to download it.) skipThisOne(); @@ -1045,25 +1040,6 @@ function syncDir(self, params, directionIsToS3) { setImmediate(checkDoMoreWork); // skip this, just a local folder } - - // different cases - - // a directory, with no remote equivalent (has been deleted) - // local=/xml remote=undefined <-- at the end. ** - // local=/xml remote=/yml/head.hbs delete it - - // a directory with a remove object equivalent - // local=/xml remote=/xml - // --> ignore it. - - // a directory, with no remote object equivalent (but a file that begins with the same name) - // local=/xml remote=/xml/something.hbs - // --> do nothing just pass on.. - return; } @@ -1107,7 +1083,6 @@ function syncDir(self, params, directionIsToS3) { if (!deleteRemoved) return; ee.deleteTotal += 1; var fullPath = path.join(localDir, localFileStat.path); - console.log('removing directory', fullPath); rimraf(fullPath, function(err) { if (fatalError) return; if (err && err.code !== 'ENOENT') return handleError(err); @@ -1123,7 +1098,6 @@ function syncDir(self, params, directionIsToS3) { if (!deleteRemoved) return; ee.deleteTotal += 1; var fullPath = path.join(localDir, localFileStat.path); - console.log('deleting:'+fullPath); fs.unlink(fullPath, function(err) { if (fatalError) return; if (err && err.code !== 'ENOENT') return handleError(err); @@ -1137,9 +1111,6 @@ function syncDir(self, params, directionIsToS3) { s3ObjectCursor += 1; setImmediate(checkDoMoreWork); var fullPath = path.join(localDir, toNativeSep(s3Object.key)); - - console.log('downloadS3Object', fullPath); - if (getS3Params) { getS3Params(fullPath, s3Object, haveS3Params); } else { @@ -1186,7 +1157,6 @@ function syncDir(self, params, directionIsToS3) { } function skipThisOne() { - console.log('skipThisOne localFile=' + allLocalFiles[localFileCursor].path + ' s3Object=' +allS3Objects[s3ObjectCursor].key); s3ObjectCursor += 1; localFileCursor += 1; setImmediate(checkDoMoreWork); @@ -1278,7 +1248,6 @@ function syncDir(self, params, directionIsToS3) { ee.emit('progress'); data.Contents.forEach(function(object) { object.key = object.Key.substring(prefix.length); - console.log('s3Object=', object.key); allS3Objects.push(object); }); checkDoMoreWork(); @@ -1309,9 +1278,6 @@ function syncDir(self, params, directionIsToS3) { } }); - for (var i = 0; i < allLocalFiles.length; i++) { - console.log('localFile=' + allLocalFiles[i].path + ', s3Path=' + allLocalFiles[i].s3Path); - } startComputingMd5Sums(); }); } From 82b22c0bf6307c741e03a720d841cc862b339038 Mon Sep 17 00:00:00 2001 From: Neal Shail Date: Thu, 8 Mar 2018 19:53:19 +0000 Subject: [PATCH 3/5] oops spaces --- lib/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/index.js b/lib/index.js index f5c68a1..11140b6 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1111,6 +1111,7 @@ function syncDir(self, params, directionIsToS3) { s3ObjectCursor += 1; setImmediate(checkDoMoreWork); var fullPath = path.join(localDir, toNativeSep(s3Object.key)); + if (getS3Params) { getS3Params(fullPath, s3Object, haveS3Params); } else { @@ -1277,7 +1278,6 @@ function syncDir(self, params, directionIsToS3) { return 0; } }); - startComputingMd5Sums(); }); } From f0d459202da3af8a7d6c5e4f9db86899ef24a6d3 Mon Sep 17 00:00:00 2001 From: Neal Shail Date: Thu, 8 Mar 2018 19:55:09 +0000 Subject: [PATCH 4/5] spaces --- lib/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/index.js b/lib/index.js index 11140b6..0ba6de7 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1111,7 +1111,7 @@ function syncDir(self, params, directionIsToS3) { s3ObjectCursor += 1; setImmediate(checkDoMoreWork); var fullPath = path.join(localDir, toNativeSep(s3Object.key)); - + if (getS3Params) { getS3Params(fullPath, s3Object, haveS3Params); } else { From 3e601e293aafbfa68fbbab2da7b3a00b1576e20f Mon Sep 17 00:00:00 2001 From: nealshail Date: Fri, 6 Dec 2019 16:37:32 +0000 Subject: [PATCH 5/5] sometimes listobjects returns folder as an object, ignore it. --- lib/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/index.js b/lib/index.js index 0ba6de7..0b93106 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1249,7 +1249,9 @@ function syncDir(self, params, directionIsToS3) { ee.emit('progress'); data.Contents.forEach(function(object) { object.key = object.Key.substring(prefix.length); - allS3Objects.push(object); + if (object.key != ""){ + allS3Objects.push(object); + } }); checkDoMoreWork(); });