From 24abfc2faafd7fd0c48ae9e50413fa3b2e91f25a Mon Sep 17 00:00:00 2001 From: Ron Waldon Date: Fri, 11 Jul 2014 11:44:36 +1000 Subject: [PATCH 1/4] update package.json dependencies --- package.json | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 885a797..6c520e4 100644 --- a/package.json +++ b/package.json @@ -36,28 +36,28 @@ "node": ">=0.9.0 <0.11.0" }, "dependencies": { - "async": "~0.2", - "aws-sdk": "~1.12", - "cli": "~0.4", - "findup-sync": "~0.1", - "glob": "~3.2", - "graceful-fs": "~2.0", - "mime": "~1.2", - "mmmagic": "~0.3", - "q": "~0.9", - "underscore": "~1.5", - "z-schema": "~2.0" + "async": "0.9.0", + "aws-sdk": "2.0.6", + "cli": "0.6.3", + "findup-sync": "0.1.3", + "glob": "4.0.3", + "graceful-fs": "3.0.2", + "mime": "1.2.11", + "mmmagic": "0.3.8", + "q": "0.9.7", + "underscore": "1.6.0", + "z-schema": "2.4.8" }, "devDependencies": { - "chai": "~1.8", + "chai": "~1.9", "grunt": "~0.4", - "grunt-contrib-watch": "~0.5", + "grunt-contrib-watch": "~0.6", "grunt-jslint": "~1.1", - "grunt-mocha-cli": "~1.3", - "grunt-mocha-cov": "~0.0.7", - "mocha": "~1.14", - "sinon": "~1.7", - "sinon-chai": "~2.4" + "grunt-mocha-cli": "~1.9", + "grunt-mocha-cov": "~0.2", + "mocha": "~1.20", + "sinon": "~1.10", + "sinon-chai": "~2.5" }, "peerDependencies": {}, "keywords": [] From 03f0c9314ed43ec1c999f7ac8586bc354948fce9 Mon Sep 17 00:00:00 2001 From: Ron Waldon Date: Fri, 11 Jul 2014 15:49:55 +1000 Subject: [PATCH 2/4] display progress bars for CDN scanning (fixes #20) --- index.js | 30 +++++++++++++++++++----------- lib/cdn/aws.js | 18 +++++++++++++++++- lib/filelist.js | 9 +++++++++ package.json | 1 + 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/index.js b/index.js index 466bc11..a32ca0d 100644 --- a/index.js +++ b/index.js @@ -9,11 +9,12 @@ path = require('path'); // 3rd-party modules -var Q, async, cli, findup; +var Q, async, cli, findup, ProgressBar; Q = require('q'); async = require('async'); cli = require('cli'); findup = require('findup-sync'); +ProgressBar = require('progress'); // custom modules @@ -74,7 +75,7 @@ function testConfig() { } function eachTarget(t, options, done) { - var actions, info; + var actions, info, theseLocalFiles, bar; info = function (msg) { cli.info(t.label + ': ' + msg); }; @@ -82,22 +83,29 @@ function eachTarget(t, options, done) { cli.info(action.toString()); }); - Q.all([ - localFiles.applyStrategy(t.strategy), - t.cdn.listFiles() - ]).spread(function (localFiles, remoteFiles) { + cli.info('applying "' + t.strategy + '" strategy to local file(s)'); + localFiles.applyStrategy(t.strategy).then(function (files) { + theseLocalFiles = files; + cli.info('scanning ' + t.label + '...'); + t.cdn.once('files.length', function (length) { + bar = new ProgressBar('[:bar] :current/:total :percent :elapsed :etas', { total: length }); + }); + t.cdn.on('file:fixed', function () { + bar.tick(); + }); + return t.cdn.listFiles(); + }).then(function (remoteFiles) { info(remoteFiles.length + ' remote file(s)'); actions = new ActionList(); - actions.compareFileLists(localFiles, remoteFiles); + actions.compareFileLists(theseLocalFiles, remoteFiles); info(actions.length + ' synchronisation action(s) to perform'); return t.cdn.executeActions(actions, options); - }).then(function () { - done(); - }).fail(function (err) { cli.fatal(err); - }).done(); + }).done(function () { + done(); + }); } function go(options) { diff --git a/lib/cdn/aws.js b/lib/cdn/aws.js index b19ee40..fb1199d 100644 --- a/lib/cdn/aws.js +++ b/lib/cdn/aws.js @@ -175,10 +175,12 @@ CDN.prototype.executeActions = function (actions, options) { * @param {File} file */ CDN.prototype.fixFile = function (file) { - var dfrd, options; + var self, dfrd, options; + self = this; dfrd = Q.defer(); if (file.mime && typeof file.mime === 'string') { + console.log(file.path + ': MIME already set'); dfrd.resolve(); return dfrd.promise; } @@ -189,6 +191,7 @@ CDN.prototype.fixFile = function (file) { }; this.api.headObject(options, function (err, res) { if (err) { + console.error(err); dfrd.reject(err); return; } @@ -196,6 +199,7 @@ CDN.prototype.fixFile = function (file) { if (res.ContentEncoding) { file.headers['Content-Encoding'] = res.ContentEncoding; } + self.emit('file:fixed'); dfrd.resolve(); }); @@ -212,8 +216,10 @@ CDN.prototype.fixFiles = function (files) { self.fixFile(file).done(done, done); }, function (err) { // onComplete if (err) { + console.err('fixFiles', error); dfrd.reject(err); } else { + console.log('fixFiles complete'); dfrd.resolve(files); } }); @@ -275,9 +281,19 @@ CDN.prototype.listFiles = function () { this.listAllFiles(files) .then(function (files) { + self.emit('files.length', files.length); return self.fixFiles(files); }) .then(function (files) { + files.sort(function (a, b) { + if (a.path < b.path) { + return -1; + } + if (a.path > b.path) { + return 1; + } + return 0; + }); dfrd.resolve(files); }) .done(); diff --git a/lib/filelist.js b/lib/filelist.js index ae8faf8..5951853 100644 --- a/lib/filelist.js +++ b/lib/filelist.js @@ -138,6 +138,15 @@ FileList.prototype.applyStrategy = function (strategy) { lists.forEach(function (list) { files = files.concat(list); }); + files.sort(function (a, b) { + if (a.path < b.path) { + return -1; + } + if (a.path > b.path) { + return 1; + } + return 0; + }); return (new FileList(files)).ready(); }); }; diff --git a/package.json b/package.json index 6c520e4..b1787d6 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "graceful-fs": "3.0.2", "mime": "1.2.11", "mmmagic": "0.3.8", + "progress": "~1.1.7", "q": "0.9.7", "underscore": "1.6.0", "z-schema": "2.4.8" From fc8ce04d2b1b3ff1ab5607bad864dbe3a82173f7 Mon Sep 17 00:00:00 2001 From: Ron Waldon Date: Fri, 11 Jul 2014 17:56:25 +1000 Subject: [PATCH 3/4] tweaks and tidy up related to #20 --- index.js | 7 ++++--- lib/cdn/aws.js | 7 +++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index a32ca0d..6b12b69 100644 --- a/index.js +++ b/index.js @@ -86,16 +86,17 @@ function eachTarget(t, options, done) { cli.info('applying "' + t.strategy + '" strategy to local file(s)'); localFiles.applyStrategy(t.strategy).then(function (files) { theseLocalFiles = files; - cli.info('scanning ' + t.label + '...'); + info('scanning...'); t.cdn.once('files.length', function (length) { - bar = new ProgressBar('[:bar] :current/:total :percent :elapsed :etas', { total: length }); + bar = new ProgressBar('[:bar] :current/:total :percent :elapsed :etas', { + total: length + }); }); t.cdn.on('file:fixed', function () { bar.tick(); }); return t.cdn.listFiles(); }).then(function (remoteFiles) { - info(remoteFiles.length + ' remote file(s)'); actions = new ActionList(); actions.compareFileLists(theseLocalFiles, remoteFiles); info(actions.length + ' synchronisation action(s) to perform'); diff --git a/lib/cdn/aws.js b/lib/cdn/aws.js index fb1199d..7e058b2 100644 --- a/lib/cdn/aws.js +++ b/lib/cdn/aws.js @@ -180,7 +180,7 @@ CDN.prototype.fixFile = function (file) { dfrd = Q.defer(); if (file.mime && typeof file.mime === 'string') { - console.log(file.path + ': MIME already set'); + self.emit('file:fixed'); dfrd.resolve(); return dfrd.promise; } @@ -212,14 +212,13 @@ CDN.prototype.fixFiles = function (files) { self = this; dfrd = Q.defer(); - async.eachLimit(files, 10, function (file, done) { // perItem + async.eachLimit(files, 15, function (file, done) { // perItem self.fixFile(file).done(done, done); }, function (err) { // onComplete if (err) { - console.err('fixFiles', error); + console.error('fixFiles', err); dfrd.reject(err); } else { - console.log('fixFiles complete'); dfrd.resolve(files); } }); From fcb05800a462e48c297c22161e870f6b9e22e6ce Mon Sep 17 00:00:00 2001 From: Ron Waldon Date: Fri, 11 Jul 2014 17:59:27 +1000 Subject: [PATCH 4/4] bump to v0.5.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b1787d6..1fe838b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cdn-sync", "description": "deflate / synchronise assets to a CDN", - "version": "0.4.0", + "version": "0.5.0", "homepage": "https://github.com/jokeyrhyme/cdn-sync", "author": { "name": "Ron Waldon",