diff -Nru node-klaw-2.0.0/CHANGELOG.md node-klaw-2.1.1/CHANGELOG.md --- node-klaw-2.0.0/CHANGELOG.md 2017-06-23 19:53:21.000000000 +0000 +++ node-klaw-2.1.1/CHANGELOG.md 2017-11-18 20:00:30.000000000 +0000 @@ -1,3 +1,15 @@ +2.1.1 / 2017-11-18 +------------------ + +- Performance optimization [#27](https://github.com/jprichardson/node-klaw/pull/27) + +2.1.0 / 2017-08-10 +------------------ + +### Added + +- Added `depthLimit` option to limit how deep to recurse into folders. [#25](https://github.com/jprichardson/node-klaw/pull/25) + 2.0.0 / 2017-06-23 ------------------ diff -Nru node-klaw-2.0.0/debian/changelog node-klaw-2.1.1/debian/changelog --- node-klaw-2.0.0/debian/changelog 2017-06-27 10:19:22.000000000 +0000 +++ node-klaw-2.1.1/debian/changelog 2017-11-20 08:24:03.000000000 +0000 @@ -1,3 +1,15 @@ +node-klaw (2.1.1-1) unstable; urgency=medium + + * New upstream release. + * Updated packaging: + - put in section javascript + - bump standards-version to 4.1.1 + - use https in d/copyright's Format field + - bump d/watch to version 4 + - update d/copyright's dates + + -- Julien Puydt Mon, 20 Nov 2017 09:24:03 +0100 + node-klaw (2.0.0-1) unstable; urgency=medium * New upstream release. diff -Nru node-klaw-2.0.0/debian/control node-klaw-2.1.1/debian/control --- node-klaw-2.0.0/debian/control 2017-06-27 10:19:22.000000000 +0000 +++ node-klaw-2.1.1/debian/control 2017-11-20 08:24:03.000000000 +0000 @@ -1,10 +1,10 @@ Source: node-klaw -Section: web +Section: javascript Priority: optional Maintainer: Debian Javascript Maintainers Uploaders: Julien Puydt Build-Depends: debhelper (>= 10), nodejs -Standards-Version: 4.0.0 +Standards-Version: 4.1.1 Homepage: https://github.com/jprichardson/node-klaw#readme Vcs-Git: https://anonscm.debian.org/git/pkg-javascript/node-klaw.git Vcs-Browser: https://anonscm.debian.org/cgit/pkg-javascript/node-klaw.git diff -Nru node-klaw-2.0.0/debian/copyright node-klaw-2.1.1/debian/copyright --- node-klaw-2.0.0/debian/copyright 2017-06-27 10:19:22.000000000 +0000 +++ node-klaw-2.1.1/debian/copyright 2017-11-20 08:24:03.000000000 +0000 @@ -1,14 +1,14 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: klaw Upstream-Contact: https://github.com/jprichardson/node-klaw/issues Source: https://github.com/jprichardson/node-klaw#readme Files: * -Copyright: 2016 JP Richardson +Copyright: 2015-2017 JP Richardson License: Expat Files: debian/* -Copyright: 2016 Julien Puydt +Copyright: 2016-2017 Julien Puydt License: Expat License: Expat diff -Nru node-klaw-2.0.0/debian/watch node-klaw-2.1.1/debian/watch --- node-klaw-2.0.0/debian/watch 2017-06-27 10:19:22.000000000 +0000 +++ node-klaw-2.1.1/debian/watch 2017-11-20 08:24:03.000000000 +0000 @@ -1,4 +1,4 @@ -version=3 +version=4 opts=\ dversionmangle=s/\+(debian|dfsg|ds|deb)(\.\d+)?$//,\ filenamemangle=s/.*\/v?([\d\.-]+)\.tar\.gz/node-klaw-$1.tar.gz/ \ diff -Nru node-klaw-2.0.0/package.json node-klaw-2.1.1/package.json --- node-klaw-2.0.0/package.json 2017-06-23 19:53:21.000000000 +0000 +++ node-klaw-2.1.1/package.json 2017-11-18 20:00:30.000000000 +0000 @@ -1,10 +1,10 @@ { "name": "klaw", - "version": "2.0.0", + "version": "2.1.1", "description": "File system walker with Readable stream interface.", "main": "./src/index.js", "scripts": { - "lint": "standard", + "lint": "standard && standard-markdown", "test": "npm run lint && npm run unit", "unit": "tape tests/**/*.js | tap-spec" }, @@ -33,6 +33,7 @@ "mkdirp": "^0.5.1", "rimraf": "^2.4.3", "standard": "^10.0.2", + "standard-markdown": "^4.0.1", "tap-spec": "^4.1.1", "tape": "^4.2.2" } diff -Nru node-klaw-2.0.0/README.md node-klaw-2.1.1/README.md --- node-klaw-2.0.0/README.md 2017-06-23 19:53:21.000000000 +0000 +++ node-klaw-2.1.1/README.md 2017-11-18 20:00:30.000000000 +0000 @@ -46,40 +46,35 @@ the following: - `queueMethod` (`string`, default: `'shift'`): Either `'shift'` or `'pop'`. On `readdir()` array, call either `shift()` or `pop()`. - `pathSorter` (`function`, default: `undefined`): Sorting [function for Arrays](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort). - - `fs` (`object`, default: `require('fs')`): Use this to hook into the `fs` methods or to use [`mock-fs`](https://github.com/tschaub/mock-fs) + - `fs` (`object`, default: [`graceful-fs`](https://github.com/isaacs/node-graceful-fs)): Use this to hook into the `fs` methods or to use [`mock-fs`](https://github.com/tschaub/mock-fs) - `filter` (`function`, default: `undefined`): Filtering [function for Arrays](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter) + - `depthLimit` (`number`, default: `undefined`): The number of times to recurse before stopping. -1 for unlimited. **Streams 1 (push) example:** ```js -var klaw = require('klaw') +const klaw = require('klaw') -var items = [] // files, directories, symlinks, etc +const items = [] // files, directories, symlinks, etc klaw('/some/dir') - .on('data', function (item) { - items.push(item.path) - }) - .on('end', function () { - console.dir(items) // => [ ... array of files] - }) + .on('data', item => items.push(item.path)) + .on('end', () => console.dir(items)) // => [ ... array of files] ``` **Streams 2 & 3 (pull) example:** ```js -var klaw = require('klaw') +const klaw = require('klaw') -var items = [] // files, directories, symlinks, etc +const items = [] // files, directories, symlinks, etc klaw('/some/dir') .on('readable', function () { - var item + let item while ((item = this.read())) { items.push(item.path) } }) - .on('end', function () { - console.dir(items) // => [ ... array of files] - }) + .on('end', () => console.dir(items)) // => [ ... array of files] ``` ### Error Handling @@ -89,22 +84,20 @@ Example: ```js -var klaw = require('klaw') +const klaw = require('klaw') + klaw('/some/dir') .on('readable', function () { - var item + let item while ((item = this.read())) { // do something with the file } }) - .on('error', function (err, item) { + .on('error', (err, item) => { console.log(err.message) console.log(item.path) // the file the error occurred on }) - .on('end', function () { - console.dir(items) // => [ ... array of files] - }) - + .on('end', () => console.dir(items)) // => [ ... array of files] ``` @@ -125,51 +118,47 @@ **Example (skipping directories):** ```js -var klaw = require('klaw') -var through2 = require('through2') +const klaw = require('klaw') +const through2 = require('through2') -var excludeDirFilter = through2.obj(function (item, enc, next) { +const excludeDirFilter = through2.obj(function (item, enc, next) { if (!item.stats.isDirectory()) this.push(item) next() }) -var items = [] // files, directories, symlinks, etc +const items = [] // files, directories, symlinks, etc klaw('/some/dir') .pipe(excludeDirFilter) - .on('data', function (item) { - items.push(item.path) - }) - .on('end', function () { - console.dir(items) // => [ ... array of files without directories] - }) - + .on('data', item => items.push(item.path)) + .on('end', () => console.dir(items)) // => [ ... array of files without directories] ``` + **Example (ignore hidden directories):** + ```js -var klaw = require('klaw') -var path = require('path') +const klaw = require('klaw') +const path = require('path') -var filterFunc = function(item){ - var basename = path.basename(item) +const filterFunc = item => { + const basename = path.basename(item) return basename === '.' || basename[0] !== '.' } -klaw('/some/dir', { filter : filterFunc }) - .on('data', function(item){ +klaw('/some/dir', { filter: filterFunc }) + .on('data', item => { // only items of none hidden folders will reach here }) - ``` **Example (totaling size of PNG files):** ```js -var klaw = require('klaw') -var path = require('path') -var through2 = require('through2') +const klaw = require('klaw') +const path = require('path') +const through2 = require('through2') -var totalPngsInBytes = 0 -var aggregatePngSize = through2.obj(function (item, enc, next) { +let totalPngsInBytes = 0 +const aggregatePngSize = through2.obj(function (item, enc, next) { if (path.extname(item.path) === '.png') { totalPngsInBytes += item.stats.size } @@ -179,23 +168,19 @@ klaw('/some/dir') .pipe(aggregatePngSize) - .on('data', function (item) { - items.push(item.path) - }) - .on('end', function () { - console.dir(totalPngsInBytes) // => total of all pngs (bytes) - }) + .on('data', item => items.push(item.path)) + .on('end', () => console.dir(totalPngsInBytes)) // => total of all pngs (bytes) ``` **Example (deleting all .tmp files):** ```js -var fs = require('fs') -var klaw = require('klaw') -var through2 = require('through2') +const fs = require('fs') +const klaw = require('klaw') +const through2 = require('through2') -var deleteAction = through2.obj(function (item, enc, next) { +const deleteAction = through2.obj(function (item, enc, next) { this.push(item) if (path.extname(item.path) === '.tmp') { @@ -204,19 +189,17 @@ } else { item.deleted = false next() - } + } }) -var deletedFiles = [] +const deletedFiles = [] klaw('/some/dir') .pipe(deleteAction) - .on('data', function (item) { + .on('data', item => { if (!item.deleted) return deletedFiles.push(item.path) }) - .on('end', function () { - console.dir(deletedFiles) // => all deleted files - }) + .on('end', () => console.dir(deletedFiles)) // => all deleted files ``` You can even chain a bunch of these filters and aggregators together. By using @@ -228,9 +211,7 @@ klaw('/some/dir') .pipe(filterCertainFiles) .pipe(deleteSomeOtherFiles) - .on('end', function () { - console.log('all done!') - }) + .on('end', () => console.log('all done!')) ``` **Example passing (piping) through errors:** @@ -239,30 +220,26 @@ `klaw` will not pipe through to the next. If you want to do this, do the following: ```js -var klaw = require('klaw') -var through2 = require('through2') +const klaw = require('klaw') +const through2 = require('through2') -var excludeDirFilter = through2.obj(function (item, enc, next) { +const excludeDirFilter = through2.obj(function (item, enc, next) { if (!item.stats.isDirectory()) this.push(item) next() }) -var items = [] // files, directories, symlinks, etc +const items = [] // files, directories, symlinks, etc klaw('/some/dir') - .on('error', function (err) { excludeDirFilter.emit('error', err) }) // forward the error on + .on('error', err => excludeDirFilter.emit('error', err)) // forward the error on .pipe(excludeDirFilter) - .on('data', function (item) { - items.push(item.path) - }) - .on('end', function () { - console.dir(items) // => [ ... array of files without directories] - }) + .on('data', item => items.push(item.path)) + .on('end', () => console.dir(items)) // => [ ... array of files without directories] ``` ### Searching Strategy -Pass in options for `queueMethod` and `pathSorter` to affect how the file system +Pass in options for `queueMethod`, `pathSorter`, and `depthLimit` to affect how the file system is recursively iterated. See the code for more details, it's less than 50 lines :) diff -Nru node-klaw-2.0.0/src/index.js node-klaw-2.1.1/src/index.js --- node-klaw-2.0.0/src/index.js 2017-06-23 19:53:21.000000000 +0000 +++ node-klaw-2.1.1/src/index.js 2017-11-18 20:00:30.000000000 +0000 @@ -6,13 +6,14 @@ function Walker (dir, options) { assert.strictEqual(typeof dir, 'string', '`dir` parameter should be of type string. Got type: ' + typeof dir) var defaultStreamOptions = { objectMode: true } - var defaultOpts = { queueMethod: 'shift', pathSorter: undefined, filter: undefined } + var defaultOpts = { queueMethod: 'shift', pathSorter: undefined, filter: undefined, depthLimit: undefined } options = Object.assign(defaultOpts, options, defaultStreamOptions) Readable.call(this, options) this.root = path.resolve(dir) this.paths = [this.root] this.options = options + if (options.depthLimit > -1) this.rootDepth = this.root.split(path.sep).length + 1 this.fs = options.fs || require('graceful-fs') } util.inherits(Walker, Readable) @@ -25,7 +26,11 @@ self.fs.lstat(pathItem, function (err, stats) { var item = { path: pathItem, stats: stats } if (err) return self.emit('error', err, item) - if (!stats.isDirectory()) return self.push(item) + + if (!stats.isDirectory() || (self.rootDepth && + pathItem.split(path.sep).length - self.rootDepth >= self.options.depthLimit)) { + return self.push(item) + } self.fs.readdir(pathItem, function (err, pathItems) { if (err) { @@ -36,7 +41,8 @@ pathItems = pathItems.map(function (part) { return path.join(pathItem, part) }) if (self.options.filter) pathItems = pathItems.filter(self.options.filter) if (self.options.pathSorter) pathItems.sort(self.options.pathSorter) - pathItems.forEach(function (pi) { self.paths.push(pi) }) + // faster way to do do incremental batch array pushes + self.paths.push.apply(self.paths, pathItems) self.push(item) }) diff -Nru node-klaw-2.0.0/tests/walk_depth-limit.test.js node-klaw-2.1.1/tests/walk_depth-limit.test.js --- node-klaw-2.0.0/tests/walk_depth-limit.test.js 1970-01-01 00:00:00.000000000 +0000 +++ node-klaw-2.1.1/tests/walk_depth-limit.test.js 2017-11-18 20:00:30.000000000 +0000 @@ -0,0 +1,60 @@ +var fs = require('fs') +var mkdirp = require('mkdirp') +var path = require('path') +var test = require('./_test') +var klaw = require('../') +var fixtures = require('./fixtures') + +test('should honor depthLimit option -1', function (t, testDir) { + var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i', + 'h/i/j', 'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg'] + run(t, testDir, -1, expected) +}) + +test('should honor depthLimit option 0', function (t, testDir) { + var expected = ['a', 'h'] + run(t, testDir, 0, expected) +}) + +test('should honor depthLimit option 1', function (t, testDir) { + var expected = ['a', 'a/b', 'a/e.jpg', 'h', 'h/i'] + run(t, testDir, 1, expected) +}) + +test('should honor depthLimit option 2', function (t, testDir) { + var expected = ['a', 'a/b', 'a/b/c', 'a/e.jpg', 'h', 'h/i', 'h/i/j', + 'h/i/l.txt', 'h/i/m.jpg'] + run(t, testDir, 2, expected) +}) + +test('should honor depthLimit option 3', function (t, testDir) { + var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i', + 'h/i/j', 'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg'] + run(t, testDir, 3, expected) +}) + +function run (t, testDir, depthLimit, expected) { + fixtures.forEach(function (f) { + f = path.join(testDir, f) + var dir = path.dirname(f) + mkdirp.sync(dir) + fs.writeFileSync(f, path.basename(f, path.extname(f))) + }) + + var items = [] + klaw(testDir, { depthLimit: depthLimit }) + .on('data', function (item) { + items.push(item.path) + }) + .on('error', t.end) + .on('end', function () { + items.sort() + expected = expected.map(function (item) { + return path.join(path.join(testDir, item)) + }) + expected.unshift(testDir) + + t.same(items, expected) + t.end() + }) +}