first commit

build/node_modules/fqueue/.travis.yml

@@ -0,0 +1,5 @@
+language: node_js
+node_js:
+  - 0.4
+  - 0.6
+  - 0.7

build/node_modules/fqueue/README.md

@@ -0,0 +1,102 @@
+Function Queue
+==============
+
+### In-memory, error handling (retry) function queue, with the ability to throttle simultaneous executions.
+
+[![Build Status](https://secure.travis-ci.org/oleics/node-fqueue.png)](http://travis-ci.org/oleics/node-fqueue)
+
+It is the distillate of [node-filewalker](https://github.com/oleics/node-filewalker)
+
+### Installation
+
+```npm install fqueue```
+
+### Usage
+
+Please have a look at the examples.
+
+Class FunctionQueue
+-------------------
+
+Inherits from events.EventEmitter
+
+### Options
+
+```scope``` (default: this)
+
+```maxPending``` (default: -1)  
+Maximum asynchronous jobs.  
+Useful to throttle the number of simultaneous disk-operations.
+
+```maxAttempts``` (default: 3)  
+Maximum reattempts on error.  
+Set to 0 to disable reattempts.  
+Set to -1 for infinite reattempts.
+
+```attemptTimeout``` (default: 5000 ms)  
+Minimum time to wait before reattempt. In milliseconds.  
+Useful to let network-drives remount, etc.
+
+### Properties
+
+.scope  
+.maxPending  
+.maxAttempts  
+.attemptTimeout
+
+.running  
+.paused  
+.pending  
+.dequeued  
+.warnings  
+.errors  
+.attempts  
+.queue
+
+### Methods
+
+this ```.start(func, args [, scope [, timeout]])```  
+Starts the function-queue.
+
+this ```.enqueue(func, args [, scope [, timeout]])```  
+Enqueues a function for later execution.
+
+this ```.done()```  
+Tell the function-queue that the function has done execution.
+
+this ```.error(err, func, args [, scope [, maxAttempts [, timeout]]])```  
+Tell the function-queue about an error. This either initiates an  
+reattempt or emits the 'error' event.  
+*Notice:* You need to call ```.done()``` even if the function  
+called ```.error([..])```.
+
+#### General Methods
+
+boolean ```.isEmpty()```  
+Returns ```true``` if the queue is empty, otherwise ```false```
+
+this ```.pause()```  
+Pauses the execution of functions. Emits the 'pause' event after  
+all pending functions completed.
+
+this ```.resume()```  
+Resumes the previously paused execution of functions. Immediately  
+emits the 'resume' event.
+
+### Events
+
+pause  
+resume  
+done  
+error err  
+retry func, args, err, r, scope
+
+## MIT License
+
+Copyright (c) 2012 Oliver Leics <oliver.leics@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

build/node_modules/fqueue/examples/du1.js

@@ -0,0 +1,69 @@
+
+var fs = require('fs'),
+    path = require('path'),
+    FunctionQueue = require('..');
+
+function Diskusage(p, cb, options) {
+  var self = this;
+  
+  var fqueue = new FunctionQueue(options);
+  var dirs = 0;
+  var files = 0;
+  var bytes = 0;
+  
+  function readdir(p) {
+    var args = arguments;
+    
+    fs.readdir(p, function(err, files) {
+      if(err) {
+        fqueue.error(err, readdir, args);
+      } else {
+        files.forEach(function(file) {
+          fqueue.enqueue(stat, [path.join(p, file)]);
+        });
+      }
+      fqueue.done();
+    });
+  }
+  
+  function stat(p) {
+    var args = arguments;
+    
+    fs.stat(p, function(err, s) {
+      if(err) {
+        fqueue.error(err, stat, args);
+      } else {
+        if(s.isDirectory()) {
+          dirs += 1;
+          fqueue.enqueue(readdir, [p]);
+        } else {
+          files += 1;
+          bytes += s.size;
+        }
+      }
+      fqueue.done();
+    });
+  }
+  
+  fqueue.on('error', function(err) {
+    console.log('error:', err);
+  });
+  
+  fqueue.on('retry', function(func, args) {
+    console.log('retry:', args[0]);
+  });
+  
+  fqueue.on('done', function() {
+    cb(bytes, dirs, files, fqueue.errors);
+  });
+  
+  fqueue.start(stat, [p], this);
+}
+
+// Runtime
+
+Diskusage('.', function(bytes, dirs, files, errors) {
+  console.log('');
+  console.log('diskusage: %d bytes in %d dirs and %d files.', bytes, dirs, files);
+  console.log('           %d unreadable paths.', errors);
+});

build/node_modules/fqueue/examples/du2.js

@@ -0,0 +1,70 @@
+
+var fs = require('fs'),
+    path = require('path'),
+    util = require('util'),
+    FunctionQueue = require('..');
+
+function Diskusage(options) {
+  FunctionQueue.call(this, options);
+  this.dirs = 0;
+  this.files = 0;
+  this.bytes = 0;
+}
+util.inherits(Diskusage, FunctionQueue);
+
+Diskusage.prototype.run = function(dir) {
+  this.start(this._stat, [dir]);
+  return this;
+};
+
+Diskusage.prototype._readdir = function(p) {
+  var self = this, args = arguments;
+  
+  fs.readdir(p, function(err, files) {
+    if(err) {
+      self.error(err, self._readdir, args);
+    } else {
+      files.forEach(function(file) {
+        self.enqueue(self._stat, [path.join(p, file)]);
+      });
+    }
+    self.done();
+  });
+};
+
+Diskusage.prototype._stat = function(p) {
+  var self = this, args = arguments;
+  
+  fs.stat(p, function(err, s) {
+    if(err) {
+      self.error(err, self._stat, args);
+    } else {
+      if(s.isDirectory()) {
+        self.dirs += 1;
+        self.enqueue(self._readdir, [p]);
+      } else {
+        self.files += 1;
+        self.bytes += s.size;
+      }
+    }
+    self.done();
+  });
+};
+
+// Runtime
+
+new Diskusage()
+  .on('error', function(err) {
+    console.log('error:', err);
+  })
+  .on('retry', function(func, args) {
+    console.log('retry:', args[0]);
+  })
+  .on('done', function() {
+    console.log('');
+    console.log('diskusage: %d bytes in %d dirs and %d files.', this.bytes, this.dirs, this.files);
+    console.log('           %d unreadable paths.', this.errors);
+  })
+.run('.');
+
+

build/node_modules/fqueue/examples/hash.js

@@ -0,0 +1,98 @@
+
+var started = Date.now();
+
+var path = require('path'),
+    fs = require('fs'),
+    createHash = require('crypto').createHash,
+    HASH_ALGO = 'md5',
+    fqueue = require('..')({
+      maxPending: 1
+    });
+    dirs = 0,
+    files = 0,
+    bytes = 0,
+    hashstack = {};
+
+fqueue.on('error', function(err) {
+  console.log('error:', err);
+});
+
+fqueue.on('retry', function(func, args) {
+  console.log('retry:', args[0]);
+});
+
+fqueue.on('done', function() {
+  console.log('');
+  var duration = Date.now()-started;
+  console.log('%d ms', duration);
+  console.log('%d dirs, %d files, %d bytes, %d errors', dirs, files, bytes, this.errors);
+  
+  var hash = createHash(HASH_ALGO),
+      keys = Object.keys(hashstack);
+  keys.sort();
+  keys.forEach(function(p) {
+    hash.update(hashstack[p]);
+  });
+  console.log('');
+  console.log(hash.digest('hex'));
+});
+
+fqueue.start(stat, ['.']);
+
+function readfile(p, s) {
+  var args = arguments,
+      rs, hash = createHash(HASH_ALGO);
+  rs = fs.createReadStream(p)
+    .on('error', function(err) {
+      fqueue.error(err, readfile, args);
+      fqueue.done();
+    })
+    .on('data', function(data) {
+      hash.update(data);
+    })
+    rs.on('end', function(data) {
+      hash = hash.digest('binary');
+      hashstack[p] = hash;
+      // console.log(new Buffer(hash, 'binary').toString('hex'), ('                '+s.size).slice(-16), p);
+      console.log(new Buffer(hash, 'binary').toString('hex'), p);
+    })
+    rs.on('close', function(data) {
+      fqueue.done();
+    })
+  ;
+}
+
+function readdir(p, s) {
+  var args = arguments;
+  
+  fs.readdir(p, function(err, files) {
+    if(err) {
+      fqueue.error(err, readdir, args);
+    } else {
+      files.forEach(function(file) {
+        fqueue.enqueue(stat, [path.join(p, file)]);
+      });
+    }
+    fqueue.done();
+  });
+}
+
+function stat(p) {
+  var args = arguments;
+  
+  fs.stat(p, function(err, s) {
+    if(err) {
+      fqueue.error(err, stat, args);
+    } else {
+      if(s.isDirectory()) {
+        dirs += 1;
+        fqueue.enqueue(readdir, [p, s]);
+      } else {
+        files += 1;
+        bytes += s.size;
+        fqueue.enqueue(readfile, [p, s]);
+      }
+    }
+    fqueue.done();
+  });
+}

build/node_modules/fqueue/fqueue.js

@@ -0,0 +1,168 @@
+
+module.exports = FunctionQueue;
+
+var util = require('util'),
+    EventEmitter = require('events').EventEmitter;
+
+var debug = false;
+
+function JobQueueReattempt(max, timeout) {
+  this.max = isNaN(max) ? 3 : max;
+  this.timeout = isNaN(timeout) ? 0 : timeout;
+  this.attempts = 0;
+}
+
+function FunctionQueue(options) {
+  if(!(this instanceof FunctionQueue)) return new FunctionQueue(options);
+  
+  var self = this;
+  
+  this.scope = this;
+  this.maxPending = -1;
+  this.maxAttempts = 3;
+  this.attemptTimeout = 5000;
+  
+  options = options || {};
+  Object.keys(options).forEach(function(k) {
+    if(self.hasOwnProperty(k)) {
+      self[k] = options[k];
+    }
+  });
+  
+  this._reset();
+}
+util.inherits(FunctionQueue, EventEmitter);
+
+FunctionQueue.prototype._reset = function() {
+  this.running = false;
+  this.paused = false;
+  this.pending = 0;
+  this.dequeued = 0;
+  this.warnings = 0;
+  this.errors = 0;
+  this.attempts = 0;
+  this.queue = [];
+  return this;
+};
+
+FunctionQueue.prototype.isEmpty = function() {
+  if(this.queue.length) {
+    return false;
+  } else {
+    return true;
+  }
+};
+
+FunctionQueue.prototype.error = function(err, func, args, scope, maxAttempts, timeout) {
+  debug&&console.log('error', func);
+  var r;
+  if(args[args.length-1] instanceof JobQueueReattempt) {
+    r = args[args.length-1];
+    r.attempts += 1;
+  } else {
+    this.warnings += 1;
+    r = new JobQueueReattempt(
+      maxAttempts != null ? maxAttempts : this.maxAttempts,
+      timeout != null ? timeout : this.attemptTimeout
+    );
+    args = Array.prototype.slice.call(args, 0);
+    args.push(r);
+  }
+  if(r.attempts === r.max) {
+    this.warnings -= 1;
+    this.errors += 1;
+    this.emit('error', err);
+  } else {
+    this.attempts += 1;
+    this.emit('retry', func, args, err, r, scope);
+    this.enqueue(func, args, scope, r.timeout);
+  }
+  return this;
+};
+
+FunctionQueue.prototype.start = function(func, args, scope, timeout) {
+  if(this.running) {
+    throw new Error('Can not start a running FunctionQueue.');
+  }
+  this._reset();
+  this.running = true;
+  this.enqueue(func, args, scope, timeout);
+  this._dequeue();
+  return this;
+};
+
+FunctionQueue.prototype.enqueue = function(func, args, scope, timeout) {
+  debug&&console.log('enqueue', func);
+  if(timeout) {
+    this.queue.push([this._timeout, this, [timeout, func, scope, args]]);
+  } else {
+    if(typeof func === 'string') { func = (scope||this.scope)[func]; }
+    this.queue.push([func, scope||this.scope, args]);
+  }
+  return this;
+};
+
+FunctionQueue.prototype._timeout = function(timeout, func, scope, args) {
+  debug&&console.log('_timeout', timeout, func);
+  var self = this;
+  setTimeout(function() {
+    self.enqueue(func, args, scope);
+    self.done();
+  }, timeout);
+};
+
+FunctionQueue.prototype._dequeue = function() {
+  debug&&console.log('_dequeue');
+  if(this.paused) {
+    return;
+  }
+  
+  while(this.maxPending <= 0 || this.pending < this.maxPending) {
+    var item = this.queue.shift();
+    if(item) {
+      this.pending += 1;
+      this.dequeued += 1;
+      item[0].apply(item[1], item[2]);
+    } else {
+      break;
+    }
+  }
+};
+
+FunctionQueue.prototype.done = function() {
+  debug&&console.log('done');
+  this.pending -= 1;
+  
+  if(this.isEmpty() === false && (this.maxPending <= 0 || this.pending < this.maxPending)) {
+    this._dequeue();
+  }
+  
+  if(this.pending === 0) {
+    if(this.paused) {
+      this.emit('pause');
+    } else {
+      this.running = false;
+      this.emit('done');
+    }
+  }
+  return this;
+};
+
+FunctionQueue.prototype.pause = function() {
+  this.paused = true;
+  return this;
+};
+
+FunctionQueue.prototype.resume = function() {
+  if(this.paused) {
+    this.paused = false;
+    if(this.isEmpty()) {
+      this.pending = 1;
+      this.done();
+    } else {
+      this._dequeue();
+      this.emit('resume');
+    }
+  }
+  return this;
+};

build/node_modules/fqueue/package.json

@@ -0,0 +1,55 @@
+{
+  "_from": "fqueue@0.0.x",
+  "_id": "fqueue@0.0.0",
+  "_inBundle": false,
+  "_integrity": "sha1-UzIFpPmtIbuqOPxhzvOpMML1SDY=",
+  "_location": "/fqueue",
+  "_phantomChildren": {},
+  "_requested": {
+    "type": "range",
+    "registry": true,
+    "raw": "fqueue@0.0.x",
+    "name": "fqueue",
+    "escapedName": "fqueue",
+    "rawSpec": "0.0.x",
+    "saveSpec": null,
+    "fetchSpec": "0.0.x"
+  },
+  "_requiredBy": [
+    "/filewalker"
+  ],
+  "_resolved": "https://registry.npmjs.org/fqueue/-/fqueue-0.0.0.tgz",
+  "_shasum": "533205a4f9ad21bbaa38fc61cef3a930c2f54836",
+  "_spec": "fqueue@0.0.x",
+  "_where": "/Users/asciidisco/Desktop/asciidisco.com/build/node_modules/filewalker",
+  "author": {
+    "name": "Oliver Leics",
+    "email": "oliver.leics@gmail.com"
+  },
+  "bugs": {
+    "url": "https://github.com/oleics/node-fqueue/issues"
+  },
+  "bundleDependencies": false,
+  "contributors": [],
+  "dependencies": {},
+  "deprecated": false,
+  "description": "In-memory, error handling (retry) function queue, with the ability to throttle simultaneous executions.",
+  "devDependencies": {
+    "mocha": "1.0.x"
+  },
+  "directories": {},
+  "engines": {
+    "node": ">= 0.4.x"
+  },
+  "homepage": "https://github.com/oleics/node-fqueue#readme",
+  "main": "./fqueue",
+  "name": "fqueue",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/oleics/node-fqueue.git"
+  },
+  "scripts": {
+    "test": "mocha -R spec"
+  },
+  "version": "0.0.0"
+}

build/node_modules/fqueue/test/basic-test.js

@@ -0,0 +1,252 @@
+
+var assert = require('assert');
+var fqueue = require('..');
+
+function enqueueRandomErrors(calls, jq, cbOk) {
+  function fn(cb) {
+    var args = arguments;
+    setTimeout(function() {
+      if(Math.floor(Math.random()*100)%2 === 0) {
+        cb(new Error(), args);
+      } else {
+        cb(null, args);
+      }
+    }, Math.floor(Math.random()*10));
+  }
+  for(var i=0; i<calls; i++) {
+    jq.enqueue(fn, [function(err, args) {
+      if(err) {
+        jq.error(err, fn, args, jq);
+      }
+      err||cbOk();
+      jq.done();
+    }], jq);
+  }
+}
+
+function enqueueFails(calls, jq, cbOk) {
+  function fn(cb) {
+    var args = arguments;
+    setTimeout(function() {
+      cb(new Error(), args);
+    }, Math.floor(Math.random()*10));
+  }
+  for(var i=0; i<calls; i++) {
+    jq.enqueue(fn, [function(err, args) {
+      if(err) {
+        jq.error(err, fn, args, jq);
+      }
+      err||cbOk();
+      jq.done();
+    }], jq);
+  }
+}
+
+function enqueueSucceeds(calls, jq, cbOk) {
+  function fn(cb) {
+    var args = arguments;
+    setTimeout(function() {
+      cb(null, args);
+    }, Math.floor(Math.random()*10));
+  }
+  for(var i=0; i<calls; i++) {
+    jq.enqueue(fn, [function(err, args) {
+      if(err) {
+        jq.error(err, fn, args, jq);
+      }
+      err||cbOk();
+      jq.done();
+    }], jq);
+  }
+}
+
+describe('FunctionQueue', function() {
+  describe('Basics', function() {
+    it('Exports one function', function() {
+      assert.ok(fqueue instanceof Function);
+    });
+    it('That function creates an instances of a FunctionQueue-class', function() {
+      assert.ok(fqueue() instanceof Object);
+      assert.ok(fqueue() instanceof fqueue);
+    });
+  });
+  
+  describe('Instances of a FunctionQueue-class', function() {
+    var jq;
+    beforeEach(function() {
+      jq = fqueue();
+    });
+    afterEach(function() {
+      jq = null;
+    });
+    
+    it('are instances of events.EventEmitter too', function() {
+      assert.ok(jq instanceof require('events').EventEmitter);
+    });
+    
+    // Properties
+    it('should have a property .scope <object> this', function() {
+      assert.ok(jq.scope!=null);
+      assert.strictEqual(typeof jq.scope, 'object');
+      assert.strictEqual(jq.scope, jq);
+    });
+    
+    it('should have a property .maxPending <number> -1', function() {
+      assert.ok(jq.maxPending!=null);
+      assert.strictEqual(typeof jq.maxPending, 'number');
+      assert.strictEqual(jq.maxPending, -1);
+    });
+    
+    it('should have a property .maxAttempts <number> 3', function() {
+      assert.ok(jq.maxAttempts!=null);
+      assert.strictEqual(typeof jq.maxAttempts, 'number');
+      assert.strictEqual(jq.maxAttempts, 3);
+    });
+    
+    it('should have a property .attemptTimeout <number> 5000', function() {
+      assert.ok(jq.attemptTimeout!=null);
+      assert.strictEqual(typeof jq.attemptTimeout, 'number');
+      assert.strictEqual(jq.attemptTimeout, 5000);
+    });
+    
+    it('should have a property .running <boolean> false', function() {
+      assert.ok(jq.running!=null);
+      assert.strictEqual(typeof jq.running, 'boolean');
+      assert.strictEqual(jq.running, false);
+    });
+    
+    it('should have a property .paused <boolean> false', function() {
+      assert.ok(jq.paused!=null);
+      assert.strictEqual(typeof jq.paused, 'boolean');
+      assert.strictEqual(jq.paused, false);
+    });
+    
+    it('should have a property .pending <number> 0', function() {
+      assert.ok(jq.pending!=null);
+      assert.strictEqual(typeof jq.pending, 'number');
+      assert.strictEqual(jq.pending, 0);
+    });
+    
+    it('should have a property .dequeued <number> 0', function() {
+      assert.ok(jq.dequeued!=null);
+      assert.strictEqual(typeof jq.dequeued, 'number');
+      assert.strictEqual(jq.dequeued, 0);
+    });
+    
+    it('should have a property .warnings <number> 0', function() {
+      assert.ok(jq.warnings!=null);
+      assert.strictEqual(typeof jq.warnings, 'number');
+      assert.strictEqual(jq.warnings, 0);
+    });
+    
+    it('should have a property .errors <number> 0', function() {
+      assert.ok(jq.errors!=null);
+      assert.strictEqual(typeof jq.errors, 'number');
+      assert.strictEqual(jq.errors, 0);
+    });
+    
+    it('should have a property .attempts <number> 0', function() {
+      assert.ok(jq.attempts!=null);
+      assert.strictEqual(typeof jq.attempts, 'number');
+      assert.strictEqual(jq.attempts, 0);
+    });
+    
+    it('should have a property .queue <array> []', function() {
+      assert.ok(jq.queue!=null);
+      assert.strictEqual(typeof jq.queue, 'object');
+      assert.ok(jq.queue instanceof Array);
+      assert.deepEqual(jq.queue, []);
+    });
+    
+    // Methods
+    it('should have a method .isEmpty()', function() {
+      assert.ok(jq.isEmpty instanceof Function);
+    });
+    
+    it('should have a method .start()', function() {
+      assert.ok(jq.start instanceof Function);
+    });
+    
+    it('should have a method .enqueue()', function() {
+      assert.ok(jq.enqueue instanceof Function);
+    });
+    
+    it('should have a method .error()', function() {
+      assert.ok(jq.error instanceof Function);
+    });
+    
+    it('should have a method .done()', function() {
+      assert.ok(jq.done instanceof Function);
+    });
+    
+    it('should have a method .pause()', function() {
+      assert.ok(jq.pause instanceof Function);
+    });
+    
+    it('should have a method .resume()', function() {
+      assert.ok(jq.resume instanceof Function);
+    });
+    
+    // 
+    describe('Usage', function() {
+      it('runs', function(done) {
+        var calls = 100;
+        var num = 0;
+        var numOk = 0;
+        var numErrors = 0;
+        var numAttempts = 0;
+        var numPause = 0;
+        var numResume = 0;
+        
+        function fnOk() {
+          numOk += 1;
+        }
+        
+        enqueueRandomErrors(calls, jq, fnOk); num += calls;
+        enqueueFails(calls, jq, fnOk); num += calls;
+        enqueueSucceeds(calls, jq, fnOk); num += calls;
+        
+        jq.on('pause', function() {
+          numPause += 1;
+          setTimeout(function() {
+            jq.resume();
+          }, 1);
+        });
+        
+        jq.on('resume', function() {
+          numResume += 1;
+          jq.pause();
+        });
+        
+        jq.on('retry', function(err) {
+          numAttempts += 1;
+        });
+        
+        jq.on('error', function(err) {
+          numErrors += 1;
+        });
+        
+        jq.on('done', function() {
+          // assert.strictEqual(num, 100);
+          assert.strictEqual(jq.pending, 0);
+          assert.strictEqual(jq.queue.length, 0);
+          assert.strictEqual(jq.attempts, numAttempts);
+          assert.strictEqual(jq.errors, numErrors);
+          assert.strictEqual(jq.errors, num-numOk);
+          assert.strictEqual(numPause-numResume, 1);
+          // console.log('calls %d, num %d, numOk %d, numErrors %d, numAttempts %d, numPause %d, numResume %d', calls, num, numOk, numErrors, numAttempts, numPause, numResume);
+          // console.log(jq);
+          done();
+        });
+        
+        jq.maxPending = calls;
+        jq.maxAttempts = 3;
+        jq.attemptTimeout = 5;
+        jq._dequeue();
+        jq.pause();
+      });
+    });
+    
+    
+  });
+});