From f66bc58ff10e2c03354d4e747f5c530e1f16e5a8 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Fri, 4 Sep 2015 18:08:34 +1200 Subject: [PATCH 01/18] starting over as fsdown --- README.md | 32 +++++++++++------------ codecs/csv.js | 11 ++------ codecs/index.js | 2 -- codecs/json.js | 11 -------- codecs/yml.js | 11 -------- index.js | 48 ++++++++++++++++++---------------- lib/parse.js | 68 ------------------------------------------------ lib/read-dir.js | 15 ----------- lib/read-file.js | 27 ------------------- package.json | 40 +++++++++++----------------- 10 files changed, 58 insertions(+), 207 deletions(-) delete mode 100644 codecs/json.js delete mode 100644 codecs/yml.js delete mode 100644 lib/parse.js delete mode 100644 lib/read-dir.js delete mode 100644 lib/read-file.js diff --git a/README.md b/README.md index 1d17d91..4b4334d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# fs-db +# fsdown -a silly **work in progress** to use the filesystem as a database. +a silly **work in progress** to use the filesystem (e.g. `.csv`, `.ldjson` files) as a database. the purpose is for apps to have human editable and readable data that can be iterated on easily (vim macros over sql migrations) and shared more openly (GitHub repos over JSON APIs). @@ -9,29 +9,27 @@ the purpose is for apps to have human editable and readable data that can be ite ### install ``` -npm install --save fs-db +npm install --save fsdown ``` ### use ``` -var FsDb = require('fs-db') +var levelup = require('levelup') -var fsDb = FsDb({ - location: __dirname + '/data', -}) +var db = levelup( + __dirname + '/things.csv', + { + db: require('fsdown')() + } +) -fsDb.createReadStream() - .pipe(process.stdout) +db.readStream() + .on('data', console.log) ``` -#### FsDb(options) +#### fsdown(codec, options) -possible `options` are: +`codec` is which codec to use (defaults to 'csv'). can be a name of an existing codec or a custom codec object, see [codecs](./codecs) for what is expected of a codec. -- `location`: root filesystem directory of the database -- `codec`: codec to use (defaults to 'json'), see [codecs](./codecs) - -#### fsDb.createReadStream() - -returns a readable [pull stream](https://npmjs.org/package/pull-stream) of objects with [JSON Pointer](https://npmjs.org/package/json-pointer) `id`s based on the path. +`options` are passed to the codec. diff --git a/codecs/csv.js b/codecs/csv.js index f84d7a0..7dbc6bb 100644 --- a/codecs/csv.js +++ b/codecs/csv.js @@ -1,11 +1,4 @@ -var csv = require('comma-separated-values') - module.exports = { - type: 'csv', - encode: function (obj) { - return csv.encode(obj, { header: true }) - }, - decode: function (str) { - return csv.parse(str, { header: true }) - } + decode: require('csv-parser'), + encode: require('json-csv').csv } diff --git a/codecs/index.js b/codecs/index.js index cfbc984..2f2a2ee 100644 --- a/codecs/index.js +++ b/codecs/index.js @@ -1,5 +1,3 @@ module.exports = { - json: require('./json'), - yml: require('./yml'), csv: require('./csv') } diff --git a/codecs/json.js b/codecs/json.js deleted file mode 100644 index ecc2f7a..0000000 --- a/codecs/json.js +++ /dev/null @@ -1,11 +0,0 @@ -var extend = require('xtend') -var encodings = require('level-codec/lib/encodings') - -module.exports = extend(encodings.json, { - encode: function encode (obj) { - return JSON.stringify(obj, null, 2) - }, - decode: function decode (str) { - return JSON.parse(str) - }, -}) diff --git a/codecs/yml.js b/codecs/yml.js deleted file mode 100644 index 5c13196..0000000 --- a/codecs/yml.js +++ /dev/null @@ -1,11 +0,0 @@ -var yaml = require('js-yaml') - -module.exports = { - type: 'yml', - encode: function (obj) { - return yaml.safeDump(obj) - }, - decode: function (str) { - return yaml.safeLoad(str) - } -} diff --git a/index.js b/index.js index f8a9bad..03f84df 100644 --- a/index.js +++ b/index.js @@ -1,34 +1,38 @@ -var pull = require('pull-stream') -var debug = require('debug')('fs-db') +var debug = require('debug')('fsdown:') +var AbstractStreamLevelDown = require('abstract-stream-leveldown') +var inherits = require('inherits') +var defined = require('defined') -var codecs = require('./codecs') +module.exports = FsDown -module.exports = FsDb - -function FsDb (options) { - if (!(this instanceof FsDb)) { - return new FsDb(options) +function FsDown (location, options) { + if (!(this instanceof FsDown)) { + return new FsDown(location, options) } - debug("constructor(", options, ")") - this.location = options.location || process.cwd() - this.fs = options.fs || require('fs') + debug("constructor(", location, options, ")") - var codec = options.codec || 'json' - this.codec = (typeof codec === 'string') ? - codecs[codec] : codec + this.location = defined(location, process.cwd()) } +inherits(FsDown, AbstractStreamLevelDown) -FsDb.prototype = { - createReadStream: createReadStream, +FsDown.prototype = { + _createReadStream: createReadStream, + _createWriteStream: createWriteStream, } function createReadStream () { debug('createReadStream()') - - return pull( - require('./lib/read-dir')(this), - require('./lib/read-file')(this), - require('./lib/parse')(this) - ) + + // get file read stream + // read file + // parse input +} + +function createWriteStream () { + debug('createWriteStream()') + + // get file write stream + // format output + // write to file } diff --git a/lib/parse.js b/lib/parse.js deleted file mode 100644 index a588e34..0000000 --- a/lib/parse.js +++ /dev/null @@ -1,68 +0,0 @@ -var isPlainObject = require('is-plain-object') -var toJsonPointer = require('json-pointer').compile -var pull = require('pull-stream') -var debug = require('debug')('fs-db:parse') - -module.exports = pull.Through(function contentParser (read, options) { - - var codec = options.codec - var ended = false - - return function parseContent (end, cb) { - read(end, function (end, file) { - if (end) { - // HACK: not sure why `ended` is necessary - if (ended) { return } - ended = true - return process.nextTick(function () { - cb(end) - }) - } - debug("file", file) - - var rootPath = file.path.split('.' + codec.type)[0] - - // TODO try catch - var content = codec.decode(file.content) - - debug("content", content) - - // identify nodes in contentObj - traverse(content, [], function (obj, path) { - - if (!obj.id && path.length === 0) { - obj.id = rootPath - } else if (!obj.id) { - obj.id = rootPath + "#" + toJsonPointer(path) - } - - debug("pushing", obj) - cb(null, obj) - - return obj.id - }) - }) - } -}) - -function traverse (obj, path, cb) { - debug("traverse", obj, path) - - if (!isPlainObject(obj)) { - return obj - } - - Object.keys(obj).forEach(function (key) { - var val = obj[key] - - if (isPlainObject(val)) { - obj[key] = traverse(val, path.concat(key), cb) - } else if (Array.isArray(val)) { - obj[key] = val.map(function (item, index) { - return traverse(item, path.concat([key, index]), cb) - }) - } - }) - - return cb(obj, path) -} diff --git a/lib/read-dir.js b/lib/read-dir.js deleted file mode 100644 index eeea1bb..0000000 --- a/lib/read-dir.js +++ /dev/null @@ -1,15 +0,0 @@ -var readdirp = require('readdirp') -var streamToPull = require('stream-to-pull-stream') -var debug = require('debug')('fs-db:read-dir') - -module.exports = function readDir (options) { - - var readdirpOptions = { - root: options.location, - fileFilter: '*.' + options.codec.type, - } - debug("readdirp", readdirpOptions) - return streamToPull.source( - readdirp(readdirpOptions) - ) -} diff --git a/lib/read-file.js b/lib/read-file.js deleted file mode 100644 index a9a9386..0000000 --- a/lib/read-file.js +++ /dev/null @@ -1,27 +0,0 @@ -var paraMap = require('pull-paramap') -var debug = require('debug')('fs-db') - -module.exports = function contentReader (options) { - - var fs = options.fs - - return paraMap(readContent) - - function readContent (entry, cb) { - debug("entry", entry) - - debug('readFile(', entry.fullPath, ')') - fs.readFile(entry.fullPath, 'utf8', function (err, content) { - debug('readFile() ->', err, content) - - if (err) { return cb(err) } - - var file = { - path: entry.path, - content: content, - } - debug('pushing', file) - cb(null, file) - }) - } -} diff --git a/package.json b/package.json index 7719606..88c7ffa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "fs-db", - "version": "2.1.3", + "name": "fsdown", + "version": "3.0.0-pre", "description": "wip to use fs as a db", "main": "index.js", "scripts": { @@ -8,43 +8,33 @@ }, "repository": { "type": "git", - "url": "https://github.com/ahdinosaur/fs-db.git" + "url": "https://github.com/holodex/fsdown.git" }, "keywords": [ "directory", - "analyze", - "walk", "json", - "yaml", - "model", + "csv", + "tsv", "fs", + "model", + "collection", "database", - "human", - "readable" + "human" ], "author": "ahdinosaur", "license": "ISC", "bugs": { - "url": "https://github.com/ahdinosaur/fs-db/issues" + "url": "https://github.com/holodex/fsdown/issues" }, - "homepage": "https://github.com/ahdinosaur/fs-db", + "homepage": "https://github.com/holodex/fsdown", "devDependencies": { - "pull-array-collate": "^0.1.0", - "sort-by": "^1.1.0", - "sort-keys": "^1.0.0", - "tape": "^3.5.0" + "tape": "^4.2.0" }, "dependencies": { - "comma-separated-values": "^3.6.2", + "abstract-stream-leveldown": "^1.0.4", "debug": "^2.1.3", - "is-plain-object": "^1.0.0", - "js-yaml": "^3.3.1", - "json-pointer": "^0.3.0", - "level-codec": "^4.2.0", - "pull-paramap": "^1.1.1", - "pull-stream": "^2.26.0", - "readdirp": "^1.3.0", - "stream-to-pull-stream": "^1.6.1", - "xtend": "^4.0.0" + "defined": "^1.0.0", + "inherits": "^2.0.1", + "readdirp": "^1.3.0" } } From d2430b59d859a81bae6772a6329f114e6a207e87 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sat, 5 Sep 2015 12:43:34 +1200 Subject: [PATCH 02/18] refresh test --- test/data.json | 32 -------------------------------- test/data/abc/stu.json | 9 --------- test/data/def/ghi/jkl/mno.json | 3 --- test/data/def/pqr.json | 9 --------- test/data/one.csv | 4 ++++ test/index.js | 25 +++++++++++-------------- 6 files changed, 15 insertions(+), 67 deletions(-) delete mode 100644 test/data.json delete mode 100644 test/data/abc/stu.json delete mode 100644 test/data/def/ghi/jkl/mno.json delete mode 100644 test/data/def/pqr.json create mode 100644 test/data/one.csv diff --git a/test/data.json b/test/data.json deleted file mode 100644 index 9d2dbf7..0000000 --- a/test/data.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "10": [ - "11", - "12", - "abc/stu#/10/2" - ], - "id": "abc/stu" - }, - { - "13": "14", - "id": "abc/stu#/10/2" - }, - { - "0": "1", - "id": "def/ghi/jkl/mno" - }, - { - "2": "3", - "4": "def/pqr#/4", - "id": "def/pqr" - }, - { - "5": "6", - "7": "def/pqr#/4/7", - "id": "def/pqr#/4" - }, - { - "8": "9", - "id": "def/pqr#/4/7" - } -] diff --git a/test/data/abc/stu.json b/test/data/abc/stu.json deleted file mode 100644 index 554026b..0000000 --- a/test/data/abc/stu.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "10": [ - "11", - "12", - { - "13": "14" - } - ] -} diff --git a/test/data/def/ghi/jkl/mno.json b/test/data/def/ghi/jkl/mno.json deleted file mode 100644 index 5af0f90..0000000 --- a/test/data/def/ghi/jkl/mno.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "0": "1" -} diff --git a/test/data/def/pqr.json b/test/data/def/pqr.json deleted file mode 100644 index 575979d..0000000 --- a/test/data/def/pqr.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "2": "3", - "4": { - "5": "6", - "7": { - "8": "9" - } - } -} diff --git a/test/data/one.csv b/test/data/one.csv new file mode 100644 index 0000000..32d81f0 --- /dev/null +++ b/test/data/one.csv @@ -0,0 +1,4 @@ +id,name,description +0,bed,a comfy couch +1,chair,a sweet spot +2,desk,a clean surface diff --git a/test/index.js b/test/index.js index 1083b4d..8830a5b 100644 --- a/test/index.js +++ b/test/index.js @@ -1,24 +1,20 @@ var test = require('tape') var fs = require('fs') var Path = require('path') -var extend = require('xtend') -var sortKeys = require('sort-keys') -var sortBy = require('sort-by') -var pull = require('pull-stream') -var pullToArray = require('pull-array-collate') -var streamToArray = require('stream-to-array') +var Abstract = require('abstract-leveldown') -var FsDb = require('../') +var FsDown = require('../') -test('Constructor', function (t) { - t.equal(typeof FsDb, 'function') - var fsDb = ctor() - t.ok(fsDb) - t.equal(typeof fsDb, 'object') - t.equal(typeof fsDb.createReadStream, 'function') +test('exports', function (t) { + t.equal(typeof FsDown, 'function') + var Ctor = FsDown() + t.equal(typeof Ctor, 'function') + var db = Ctor() + t.ok(Abstract.isLevelDown(db)) t.end() }) +/* test('.createReadStream()', function (t) { var fsDb = ctor() var readStream = fsDb.createReadStream() @@ -26,7 +22,7 @@ test('.createReadStream()', function (t) { readStream, pullToArray(), pull.drain(function (data) { - var expected = readData('./data.json') + var expected = readData('./data/.json') var actual = sortData(data) t.deepEqual(actual, expected) }, function (err) { @@ -55,3 +51,4 @@ function sortData (data) { return sortKeys(item) }).sort(sortBy('id')) } +*/ From c42ae69b72796d98ee2e601153614cc5719f411a Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sat, 5 Sep 2015 12:43:59 +1200 Subject: [PATCH 03/18] just keep iterating --- codecs/csv.js | 4 +++- index.js | 39 ++++++++++++++++++++++++++------------- package.json | 4 +++- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/codecs/csv.js b/codecs/csv.js index 7dbc6bb..160d13a 100644 --- a/codecs/csv.js +++ b/codecs/csv.js @@ -1,4 +1,6 @@ module.exports = { decode: require('csv-parser'), - encode: require('json-csv').csv + encode: function () { + throw new Error('FsDown: csv.encode not implemented!') + } } diff --git a/index.js b/index.js index 03f84df..237bf52 100644 --- a/index.js +++ b/index.js @@ -1,24 +1,37 @@ var debug = require('debug')('fsdown:') -var AbstractStreamLevelDown = require('abstract-stream-leveldown') -var inherits = require('inherits') +var stampit = require('stampit') +var AbstractDown = stampit.convertConstructor(require('abstract-stream-leveldown')) var defined = require('defined') -module.exports = FsDown +var codecs = require('./codecs') -function FsDown (location, options) { - if (!(this instanceof FsDown)) { - return new FsDown(location, options) - } +module.exports = configureFsDown + +var FsDown = stampit({ + props: { + _createReadStream: createReadStream, + _createWriteStream: createWriteStream, + }, +}).compose(AbstractDown) - debug("constructor(", location, options, ")") +function configureFsDown (options) { + options = defined(options, {}) - this.location = defined(location, process.cwd()) + return FsDown.props({ + codec: getCodec(options), + options: options + }) } -inherits(FsDown, AbstractStreamLevelDown) -FsDown.prototype = { - _createReadStream: createReadStream, - _createWriteStream: createWriteStream, +module.exports = configureFsDown + +function getCodec (options) { + var codec = options.codec + if (typeof codec === 'string') { + codec = codecs[codec] + } else if (!codec || typeof codec !== 'object') { + codec = codecs.csv + } } function createReadStream () { diff --git a/package.json b/package.json index 88c7ffa..264d2e8 100644 --- a/package.json +++ b/package.json @@ -28,11 +28,13 @@ }, "homepage": "https://github.com/holodex/fsdown", "devDependencies": { + "abstract-leveldown": "^2.4.1", "tape": "^4.2.0" }, "dependencies": { "abstract-stream-leveldown": "^1.0.4", - "debug": "^2.1.3", + "csv-parser": "^1.7.0", + "debug": "^2.2.0", "defined": "^1.0.0", "inherits": "^2.0.1", "readdirp": "^1.3.0" From 34268baf80e4a0fa08f504d11260639294a8c999 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sat, 5 Sep 2015 15:42:49 +1200 Subject: [PATCH 04/18] setup export interface, use forked version of abstract-stream-leveldown --- index.js | 46 ++++++++++++++++++++++++++++++---------------- package.json | 3 ++- test/index.js | 14 +++++++++----- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/index.js b/index.js index 237bf52..579a71b 100644 --- a/index.js +++ b/index.js @@ -1,29 +1,43 @@ var debug = require('debug')('fsdown:') -var stampit = require('stampit') -var AbstractDown = stampit.convertConstructor(require('abstract-stream-leveldown')) +var AbstractDown = require('abstract-stream-leveldown').AbstractStreamLevelDOWN var defined = require('defined') +var inherits = require('inherits') +var assign = require('object-assign') var codecs = require('./codecs') -module.exports = configureFsDown +module.exports = getFsDownCtor -var FsDown = stampit({ - props: { - _createReadStream: createReadStream, - _createWriteStream: createWriteStream, - }, -}).compose(AbstractDown) +function FsDown (location) { + if (!(this instanceof FsDown)) + return new FsDown(location) -function configureFsDown (options) { + AbstractDown.call(this, location) +} +inherits(FsDown, AbstractDown) + +assign(FsDown.prototype, { + _createReadStream: createReadStream, + _createWriteStream: createWriteStream +}) + +function getFsDownCtor (options) { options = defined(options, {}) - return FsDown.props({ - codec: getCodec(options), - options: options - }) -} + var codec = getCodec(options) -module.exports = configureFsDown + function FsDownCtor (location) { + if (!(this instanceof FsDownCtor)) + return new FsDownCtor(location) + + this.codec = codec + this.options = options + FsDown.call(this, location) + } + inherits(FsDownCtor, FsDown) + + return FsDownCtor +} function getCodec (options) { var codec = options.codec diff --git a/package.json b/package.json index 264d2e8..365e206 100644 --- a/package.json +++ b/package.json @@ -32,11 +32,12 @@ "tape": "^4.2.0" }, "dependencies": { - "abstract-stream-leveldown": "^1.0.4", + "abstract-stream-leveldown": "git://github.com/holodex/abstract-stream-leveldown", "csv-parser": "^1.7.0", "debug": "^2.2.0", "defined": "^1.0.0", "inherits": "^2.0.1", + "object-assign": "^4.0.1", "readdirp": "^1.3.0" } } diff --git a/test/index.js b/test/index.js index 8830a5b..a3840a2 100644 --- a/test/index.js +++ b/test/index.js @@ -5,15 +5,19 @@ var Abstract = require('abstract-leveldown') var FsDown = require('../') -test('exports', function (t) { +test('exports proper api', function (t) { t.equal(typeof FsDown, 'function') - var Ctor = FsDown() - t.equal(typeof Ctor, 'function') - var db = Ctor() - t.ok(Abstract.isLevelDown(db)) + var Db = FsDown() + t.equal(typeof Db, 'function') + var db = Db('data/one.csv') + t.ok(Abstract.isLevelDOWN(db)) t.end() }) +// TODO use abstract-leveldown tests +// https://github.com/Level/abstract-leveldown/blob/master/test.js +// https://github.com/calvinmetcalf/SQLdown/blob/master/test/test.js + /* test('.createReadStream()', function (t) { var fsDb = ctor() From 785d6f0b44becea94eb471e44f7a7fa97bd57de7 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sat, 5 Sep 2015 17:02:38 +1200 Subject: [PATCH 05/18] createReadStream --- index.js | 41 ++++++++++++++++++++++++++++++----------- package.json | 8 +++++++- test/data/one.csv | 2 +- test/data/one.json | 26 ++++++++++++++++++++++++++ test/index.js | 43 +++++++++++++++++++++---------------------- 5 files changed, 85 insertions(+), 35 deletions(-) create mode 100644 test/data/one.json diff --git a/index.js b/index.js index 579a71b..b50f551 100644 --- a/index.js +++ b/index.js @@ -1,8 +1,11 @@ -var debug = require('debug')('fsdown:') -var AbstractDown = require('abstract-stream-leveldown').AbstractStreamLevelDOWN +var debug = require('debug')('fsdown') +var fs = require('fs') var defined = require('defined') var inherits = require('inherits') var assign = require('object-assign') +var AbstractDown = require('abstract-stream-leveldown').AbstractStreamLevelDOWN +var through = require('through2') +var uuid = require('node-uuid') var codecs = require('./codecs') @@ -21,17 +24,18 @@ assign(FsDown.prototype, { _createWriteStream: createWriteStream }) -function getFsDownCtor (options) { +function getFsDownCtor (codec, options) { + codec = getCodec(codec) options = defined(options, {}) - var codec = getCodec(options) - function FsDownCtor (location) { if (!(this instanceof FsDownCtor)) return new FsDownCtor(location) this.codec = codec this.options = options + this.keyAttribute = defined(options.keyAttribute, 'key') + FsDown.call(this, location) } inherits(FsDownCtor, FsDown) @@ -39,21 +43,36 @@ function getFsDownCtor (options) { return FsDownCtor } -function getCodec (options) { - var codec = options.codec +function getCodec (codec) { if (typeof codec === 'string') { codec = codecs[codec] - } else if (!codec || typeof codec !== 'object') { + } else if (codec == null || typeof codec !== 'object') { codec = codecs.csv } + return codec } function createReadStream () { debug('createReadStream()') - // get file read stream - // read file - // parse input + var keyAttribute = this.keyAttribute + + // read from file + return fs.createReadStream(this.location) + // parse input + .pipe(this.codec.decode(this.options)) + .pipe(through.obj(function (row, enc, cb) { + var key = row[keyAttribute] + + if (key == null) { + key = uuid() + } + + cb(null, { + key: row[keyAttribute], + value: row + }) + })) } function createWriteStream () { diff --git a/package.json b/package.json index 365e206..fd5b80d 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,10 @@ "homepage": "https://github.com/holodex/fsdown", "devDependencies": { "abstract-leveldown": "^2.4.1", + "levelup": "^1.2.1", + "sort-by": "^1.1.1", + "sort-keys": "^1.1.1", + "stream-to-array": "^2.0.2", "tape": "^4.2.0" }, "dependencies": { @@ -37,7 +41,9 @@ "debug": "^2.2.0", "defined": "^1.0.0", "inherits": "^2.0.1", + "node-uuid": "^1.4.3", "object-assign": "^4.0.1", - "readdirp": "^1.3.0" + "readdirp": "^1.3.0", + "through2": "^2.0.0" } } diff --git a/test/data/one.csv b/test/data/one.csv index 32d81f0..bf9947e 100644 --- a/test/data/one.csv +++ b/test/data/one.csv @@ -1,4 +1,4 @@ -id,name,description +key,name,description 0,bed,a comfy couch 1,chair,a sweet spot 2,desk,a clean surface diff --git a/test/data/one.json b/test/data/one.json new file mode 100644 index 0000000..7d86ae8 --- /dev/null +++ b/test/data/one.json @@ -0,0 +1,26 @@ +[ + { + "key": "0", + "value": { + "key": "0", + "name": "bed", + "description": "a comfy couch" + } + }, + { + "key": "1", + "value": { + "key": "1", + "name": "chair", + "description": "a sweet spot" + } + }, + { + "key": "2", + "value": { + "key": "2", + "name": "desk", + "description": "a clean surface" + } + } +] diff --git a/test/index.js b/test/index.js index a3840a2..8b75488 100644 --- a/test/index.js +++ b/test/index.js @@ -1,7 +1,11 @@ var test = require('tape') var fs = require('fs') var Path = require('path') +var toArray = require('stream-to-array') +var sortKeys = require('sort-keys') +var sortBy = require('sort-by') var Abstract = require('abstract-leveldown') +var levelup = require('levelup') var FsDown = require('../') @@ -18,34 +22,30 @@ test('exports proper api', function (t) { // https://github.com/Level/abstract-leveldown/blob/master/test.js // https://github.com/calvinmetcalf/SQLdown/blob/master/test/test.js -/* -test('.createReadStream()', function (t) { - var fsDb = ctor() - var readStream = fsDb.createReadStream() - pull( - readStream, - pullToArray(), - pull.drain(function (data) { - var expected = readData('./data/.json') - var actual = sortData(data) - t.deepEqual(actual, expected) - }, function (err) { - t.notOk(err, 'no error') - t.end() - }) - ) +test('csv .createReadStream()', function (t) { + var db = ctor('one.csv', 'csv') + var readStream = db.createReadStream() + toArray(readStream, function (err, data) { + t.error(err, 'no error') + var expected = readData('one.json') + t.deepEqual(data, expected, 'data is correct') + t.end() + }) }) -function ctor (options) { - return FsDb(extend({ - location: __dirname + '/data', - }, options || {})) +function ctor (location, codec, options) { + return levelup( + Path.join(__dirname, 'data', location), + { + db: FsDown(codec, options) + } + ) } function readData (file) { return JSON.parse( fs.readFileSync( - Path.join(__dirname, file), 'utf8' + Path.join(__dirname, 'data', file), 'utf8' ) ) } @@ -55,4 +55,3 @@ function sortData (data) { return sortKeys(item) }).sort(sortBy('id')) } -*/ From ef55652a3da7121d75050518002aaf502344337f Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 6 Sep 2015 20:32:19 +1200 Subject: [PATCH 06/18] use csv-formatter module as csv codec.encode --- codecs/csv.js | 4 +--- package.json | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/codecs/csv.js b/codecs/csv.js index 160d13a..33cf56d 100644 --- a/codecs/csv.js +++ b/codecs/csv.js @@ -1,6 +1,4 @@ module.exports = { decode: require('csv-parser'), - encode: function () { - throw new Error('FsDown: csv.encode not implemented!') - } + encode: require('csv-formatter') } diff --git a/package.json b/package.json index fd5b80d..8f51e29 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ }, "dependencies": { "abstract-stream-leveldown": "git://github.com/holodex/abstract-stream-leveldown", + "csv-formatter": "^1.0.0", "csv-parser": "^1.7.0", "debug": "^2.2.0", "defined": "^1.0.0", From 14d1ef04526c06565ac0b0b4b2c362d92936a2a3 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 6 Sep 2015 20:32:45 +1200 Subject: [PATCH 07/18] attempt db.createWritableStream --- index.js | 66 +++++++++++++++++++++++++++++++++++++++------------ package.json | 10 +++++--- test/index.js | 29 +++++++++++++++++----- 3 files changed, 81 insertions(+), 24 deletions(-) diff --git a/index.js b/index.js index b50f551..ce397cd 100644 --- a/index.js +++ b/index.js @@ -2,10 +2,13 @@ var debug = require('debug')('fsdown') var fs = require('fs') var defined = require('defined') var inherits = require('inherits') -var assign = require('object-assign') +var assign = require('lodash.assign') +var forEach = require('lodash.foreach') var AbstractDown = require('abstract-stream-leveldown').AbstractStreamLevelDOWN var through = require('through2') var uuid = require('node-uuid') +var combine = require('stream-combiner2') +var prepend = require('prepend-stream') var codecs = require('./codecs') @@ -24,17 +27,20 @@ assign(FsDown.prototype, { _createWriteStream: createWriteStream }) -function getFsDownCtor (codec, options) { - codec = getCodec(codec) +function getFsDownCtor (options, codecOptions) { options = defined(options, {}) + codecOptions = defined(codecOptions, {}) + + var codec = getCodec(options.codec) + var keyAttribute = defined(options.keyAttribute, 'key') function FsDownCtor (location) { if (!(this instanceof FsDownCtor)) return new FsDownCtor(location) this.codec = codec - this.options = options - this.keyAttribute = defined(options.keyAttribute, 'key') + this.codecOptions = codecOptions + this.keyAttribute = keyAttribute FsDown.call(this, location) } @@ -52,15 +58,15 @@ function getCodec (codec) { return codec } -function createReadStream () { - debug('createReadStream()') +function createReadStream (options) { + debug('createReadStream(', options, ')') var keyAttribute = this.keyAttribute // read from file return fs.createReadStream(this.location) // parse input - .pipe(this.codec.decode(this.options)) + .pipe(this.codec.decode(this.codecOptions)) .pipe(through.obj(function (row, enc, cb) { var key = row[keyAttribute] @@ -69,16 +75,46 @@ function createReadStream () { } cb(null, { - key: row[keyAttribute], + key: key, value: row }) })) } -function createWriteStream () { - debug('createWriteStream()') - - // get file write stream - // format output - // write to file +function createWriteStream (options) { + debug('createWriteStream(', options, ')') + + var table = {} + + // write current content + // plus additional changes + return combine.obj([ + through.obj(function (row, enc, cb) { + row.value = JSON.parse(row.value) + cb(null, row) + }), + prepend.obj(this._createReadStream(options)), + through.obj( + function transform (row, enc, cb) { + debug("transform row", row) + // construct in-memory table + if (row.type === 'del') { + delete table[row.key] + } else { + table[row.key] = row.value + } + cb() + }, + function flush (cb) { + // output in-memory table + debug("flush table", table) + forEach(table, function (value, key) { + this.push(value) + }, this) + cb() + } + ), + this.codec.encode(this.codecOptions), + fs.createWriteStream(this.location) + ]) } diff --git a/package.json b/package.json index 8f51e29..229f70c 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,11 @@ "homepage": "https://github.com/holodex/fsdown", "devDependencies": { "abstract-leveldown": "^2.4.1", + "level-ws": "0.0.0", "levelup": "^1.2.1", "sort-by": "^1.1.1", "sort-keys": "^1.1.1", + "stream-array": "^1.1.1", "stream-to-array": "^2.0.2", "tape": "^4.2.0" }, @@ -42,9 +44,11 @@ "debug": "^2.2.0", "defined": "^1.0.0", "inherits": "^2.0.1", + "lodash.assign": "^3.2.0", + "lodash.foreach": "^3.0.3", "node-uuid": "^1.4.3", - "object-assign": "^4.0.1", - "readdirp": "^1.3.0", - "through2": "^2.0.0" + "stream-combiner2": "git://github.com/zertosh/stream-combiner2#streams3", + "through2": "^2.0.0", + "write-only-stream": "^1.1.0" } } diff --git a/test/index.js b/test/index.js index 8b75488..2c8518b 100644 --- a/test/index.js +++ b/test/index.js @@ -1,11 +1,13 @@ var test = require('tape') var fs = require('fs') var Path = require('path') +var toStream = require('stream-array') var toArray = require('stream-to-array') var sortKeys = require('sort-keys') var sortBy = require('sort-by') var Abstract = require('abstract-leveldown') var levelup = require('levelup') +var levelws = require('level-ws') var FsDown = require('../') @@ -33,21 +35,36 @@ test('csv .createReadStream()', function (t) { }) }) +test('csv .createWriteStream()', function (t) { + var db = ctor('two.csv', 'csv') + toStream(readData('one.json')) + .pipe(db.createWriteStream({ valueEncoding: 'json' })) + .on('finish', function () { + t.deepEqual( + readFile('two.csv'), + readFile('one.csv') + ) + t.end() + }) +}) + function ctor (location, codec, options) { - return levelup( + return levelws(levelup( Path.join(__dirname, 'data', location), { db: FsDown(codec, options) } + )) +} + +function readFile (file) { + return fs.readFileSync( + Path.join(__dirname, 'data', file), 'utf8' ) } function readData (file) { - return JSON.parse( - fs.readFileSync( - Path.join(__dirname, 'data', file), 'utf8' - ) - ) + return JSON.parse(readFile(file)) } function sortData (data) { From 3b5acac2fce385d25eca0f5d24a026b7634b4536 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 6 Sep 2015 22:57:59 +1200 Subject: [PATCH 08/18] use pumpify instead of stream-combiner2 --- index.js | 4 ++-- package.json | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index ce397cd..23d203b 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ var forEach = require('lodash.foreach') var AbstractDown = require('abstract-stream-leveldown').AbstractStreamLevelDOWN var through = require('through2') var uuid = require('node-uuid') -var combine = require('stream-combiner2') +var pumpify = require('pumpify') var prepend = require('prepend-stream') var codecs = require('./codecs') @@ -88,7 +88,7 @@ function createWriteStream (options) { // write current content // plus additional changes - return combine.obj([ + return pumpify.obj([ through.obj(function (row, enc, cb) { row.value = JSON.parse(row.value) cb(null, row) diff --git a/package.json b/package.json index 229f70c..25ad50f 100644 --- a/package.json +++ b/package.json @@ -47,8 +47,7 @@ "lodash.assign": "^3.2.0", "lodash.foreach": "^3.0.3", "node-uuid": "^1.4.3", - "stream-combiner2": "git://github.com/zertosh/stream-combiner2#streams3", - "through2": "^2.0.0", - "write-only-stream": "^1.1.0" + "pumpify": "^1.0.2", + "through2": "^2.0.0" } } From b93b2f7bec84bc50b44338fee40b5b12cef04916 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 6 Sep 2015 23:03:30 +1200 Subject: [PATCH 09/18] comments --- index.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 23d203b..4ad70a9 100644 --- a/index.js +++ b/index.js @@ -65,11 +65,13 @@ function createReadStream (options) { // read from file return fs.createReadStream(this.location) - // parse input + // parse data into objects .pipe(this.codec.decode(this.codecOptions)) .pipe(through.obj(function (row, enc, cb) { + // get key var key = row[keyAttribute] + // if no key, default to UUID if (key == null) { key = uuid() } @@ -86,18 +88,20 @@ function createWriteStream (options) { var table = {} - // write current content - // plus additional changes return pumpify.obj([ + // parse values as json through.obj(function (row, enc, cb) { row.value = JSON.parse(row.value) cb(null, row) }), + // add current data to beginning of + // the data that is to be written prepend.obj(this._createReadStream(options)), + // construct in-memory table of data through.obj( function transform (row, enc, cb) { debug("transform row", row) - // construct in-memory table + // perform operation to table if (row.type === 'del') { delete table[row.key] } else { @@ -106,7 +110,7 @@ function createWriteStream (options) { cb() }, function flush (cb) { - // output in-memory table + // output contents of table debug("flush table", table) forEach(table, function (value, key) { this.push(value) @@ -114,7 +118,9 @@ function createWriteStream (options) { cb() } ), + // format data to string this.codec.encode(this.codecOptions), + // write to file fs.createWriteStream(this.location) ]) } From 01e04bbce0c61363a9314235cb4e4931a5f9bb9b Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 6 Sep 2015 23:03:34 +1200 Subject: [PATCH 10/18] update README --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4b4334d..f25aca1 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,11 @@ db.readStream() .on('data', console.log) ``` -#### fsdown(codec, options) +#### fsdown(options, codecOptions) -`codec` is which codec to use (defaults to 'csv'). can be a name of an existing codec or a custom codec object, see [codecs](./codecs) for what is expected of a codec. +`options`: -`options` are passed to the codec. +- `codec` is which codec to use (defaults to 'csv'). can be a name of an existing codec or a custom codec object, see [codecs](./codecs) for what is expected of a codec. +- `keyAttribute` is a string identifier of the attribute used as keys (e.g. 'id'). + +`codecOptions` are passed to the codec. From 357e225fc56827291917a2dec2f57b06b1e9e6c3 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Mon, 7 Sep 2015 19:39:16 +1200 Subject: [PATCH 11/18] npm i --save holodex/prepend-stream --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 25ad50f..469e1a1 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "lodash.assign": "^3.2.0", "lodash.foreach": "^3.0.3", "node-uuid": "^1.4.3", + "prepend-stream": "holodex/prepend-stream", "pumpify": "^1.0.2", "through2": "^2.0.0" } From b156d1dac977bf61a4eaf21ac75e08d0333bcf7d Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Fri, 11 Sep 2015 16:05:34 +1200 Subject: [PATCH 12/18] got enough of my fork merged in to work --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 469e1a1..c2fb112 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "tape": "^4.2.0" }, "dependencies": { - "abstract-stream-leveldown": "git://github.com/holodex/abstract-stream-leveldown", + "abstract-stream-leveldown": "^1.1.1", "csv-formatter": "^1.0.0", "csv-parser": "^1.7.0", "debug": "^2.2.0", From 15ac93ec01877b4b8a246a1e295618b38bfcb6cc Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Fri, 11 Sep 2015 17:23:57 +1200 Subject: [PATCH 13/18] stringify on read --- index.js | 2 +- test/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 4ad70a9..c78984e 100644 --- a/index.js +++ b/index.js @@ -78,7 +78,7 @@ function createReadStream (options) { cb(null, { key: key, - value: row + value: JSON.stringify(row) }) })) } diff --git a/test/index.js b/test/index.js index 2c8518b..d5a123c 100644 --- a/test/index.js +++ b/test/index.js @@ -26,7 +26,7 @@ test('exports proper api', function (t) { test('csv .createReadStream()', function (t) { var db = ctor('one.csv', 'csv') - var readStream = db.createReadStream() + var readStream = db.createReadStream({ valueEncoding: 'json' }) toArray(readStream, function (err, data) { t.error(err, 'no error') var expected = readData('one.json') From 16a3e88e46c6c4f6bf09573a47bffda44b53fa88 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Mon, 21 Sep 2015 17:38:58 +1200 Subject: [PATCH 14/18] back to fs-db ref https://github.com/holodex/fsdown/pull/3#issuecomment-141387404 - drop support for leveldown - export prototype with create{Read,Write}Stream --- README.md | 14 +++++------ index.js | 64 +++++++++++++++++++-------------------------------- package.json | 12 ++++------ test/index.js | 33 ++++++++++---------------- 4 files changed, 47 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index f25aca1..00f804c 100644 --- a/README.md +++ b/README.md @@ -9,26 +9,26 @@ the purpose is for apps to have human editable and readable data that can be ite ### install ``` -npm install --save fsdown +npm install --save fs-db ``` ### use ``` -var levelup = require('levelup') +var FsDb = require('fs-db') -var db = levelup( +var db = FsDb( __dirname + '/things.csv', - { - db: require('fsdown')() - } + 'csv' ) db.readStream() .on('data', console.log) ``` -#### fsdown(options, codecOptions) +#### fsdown(location, options, codecOptions) + +`location` is the path to the database file. `options`: diff --git a/index.js b/index.js index c78984e..1f08271 100644 --- a/index.js +++ b/index.js @@ -4,7 +4,6 @@ var defined = require('defined') var inherits = require('inherits') var assign = require('lodash.assign') var forEach = require('lodash.foreach') -var AbstractDown = require('abstract-stream-leveldown').AbstractStreamLevelDOWN var through = require('through2') var uuid = require('node-uuid') var pumpify = require('pumpify') @@ -12,43 +11,33 @@ var prepend = require('prepend-stream') var codecs = require('./codecs') -module.exports = getFsDownCtor +module.exports = FsDb -function FsDown (location) { - if (!(this instanceof FsDown)) - return new FsDown(location) +function FsDb (location, options, codecOptions) { + if (!(this instanceof FsDb)) + return new FsDb(location, options, codecOptions) - AbstractDown.call(this, location) -} -inherits(FsDown, AbstractDown) - -assign(FsDown.prototype, { - _createReadStream: createReadStream, - _createWriteStream: createWriteStream -}) - -function getFsDownCtor (options, codecOptions) { - options = defined(options, {}) - codecOptions = defined(codecOptions, {}) - - var codec = getCodec(options.codec) - var keyAttribute = defined(options.keyAttribute, 'key') - - function FsDownCtor (location) { - if (!(this instanceof FsDownCtor)) - return new FsDownCtor(location) - - this.codec = codec - this.codecOptions = codecOptions - this.keyAttribute = keyAttribute + if (typeof location === 'undefined') { + throw new Error('fs-db: location is not defined') + } - FsDown.call(this, location) + if (typeof options === 'string') { + options = { codec: options } + } else { + options = defined(options, {}) } - inherits(FsDownCtor, FsDown) - return FsDownCtor + this.location = location + this.codec = getCodec(options.codec) + this.codecOptions = defined(codecOptions, {}) + this.keyAttribute = defined(options.keyAttribute, 'key') } +assign(FsDb.prototype, { + createReadStream: createReadStream, + createWriteStream: createWriteStream +}) + function getCodec (codec) { if (typeof codec === 'string') { codec = codecs[codec] @@ -76,10 +65,7 @@ function createReadStream (options) { key = uuid() } - cb(null, { - key: key, - value: JSON.stringify(row) - }) + cb(null, { key: key, value: row }) })) } @@ -90,13 +76,11 @@ function createWriteStream (options) { return pumpify.obj([ // parse values as json - through.obj(function (row, enc, cb) { - row.value = JSON.parse(row.value) - cb(null, row) - }), // add current data to beginning of // the data that is to be written - prepend.obj(this._createReadStream(options)), + prepend.obj( + this.createReadStream(options) + ), // construct in-memory table of data through.obj( function transform (row, enc, cb) { diff --git a/package.json b/package.json index c2fb112..226410b 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "fsdown", + "name": "fs-db", "version": "3.0.0-pre", "description": "wip to use fs as a db", "main": "index.js", @@ -8,7 +8,7 @@ }, "repository": { "type": "git", - "url": "https://github.com/holodex/fsdown.git" + "url": "https://github.com/holodex/fs-db.git" }, "keywords": [ "directory", @@ -24,13 +24,10 @@ "author": "ahdinosaur", "license": "ISC", "bugs": { - "url": "https://github.com/holodex/fsdown/issues" + "url": "https://github.com/holodex/fs-db/issues" }, - "homepage": "https://github.com/holodex/fsdown", + "homepage": "https://github.com/holodex/fs-db", "devDependencies": { - "abstract-leveldown": "^2.4.1", - "level-ws": "0.0.0", - "levelup": "^1.2.1", "sort-by": "^1.1.1", "sort-keys": "^1.1.1", "stream-array": "^1.1.1", @@ -38,7 +35,6 @@ "tape": "^4.2.0" }, "dependencies": { - "abstract-stream-leveldown": "^1.1.1", "csv-formatter": "^1.0.0", "csv-parser": "^1.7.0", "debug": "^2.2.0", diff --git a/test/index.js b/test/index.js index d5a123c..7d06ebe 100644 --- a/test/index.js +++ b/test/index.js @@ -5,28 +5,21 @@ var toStream = require('stream-array') var toArray = require('stream-to-array') var sortKeys = require('sort-keys') var sortBy = require('sort-by') -var Abstract = require('abstract-leveldown') -var levelup = require('levelup') -var levelws = require('level-ws') -var FsDown = require('../') +var FsDb = require('../') test('exports proper api', function (t) { - t.equal(typeof FsDown, 'function') - var Db = FsDown() - t.equal(typeof Db, 'function') - var db = Db('data/one.csv') - t.ok(Abstract.isLevelDOWN(db)) + t.equal(typeof FsDb, 'function') + var db = FsDb('data/one.csv') + t.equal(typeof db, 'object') + t.equal(typeof db.createReadStream, 'function') + t.equal(typeof db.createWriteStream, 'function') t.end() }) -// TODO use abstract-leveldown tests -// https://github.com/Level/abstract-leveldown/blob/master/test.js -// https://github.com/calvinmetcalf/SQLdown/blob/master/test/test.js - test('csv .createReadStream()', function (t) { var db = ctor('one.csv', 'csv') - var readStream = db.createReadStream({ valueEncoding: 'json' }) + var readStream = db.createReadStream() toArray(readStream, function (err, data) { t.error(err, 'no error') var expected = readData('one.json') @@ -38,7 +31,7 @@ test('csv .createReadStream()', function (t) { test('csv .createWriteStream()', function (t) { var db = ctor('two.csv', 'csv') toStream(readData('one.json')) - .pipe(db.createWriteStream({ valueEncoding: 'json' })) + .pipe(db.createWriteStream()) .on('finish', function () { t.deepEqual( readFile('two.csv'), @@ -48,13 +41,11 @@ test('csv .createWriteStream()', function (t) { }) }) -function ctor (location, codec, options) { - return levelws(levelup( +function ctor (location, options, codecOptions) { + return FsDb( Path.join(__dirname, 'data', location), - { - db: FsDown(codec, options) - } - )) + options, codecOptions + ) } function readFile (file) { From fa060743917966ac532b22b56c7330143d01abee Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Mon, 21 Sep 2015 19:59:42 +1200 Subject: [PATCH 15/18] simplify api to take just a single options object --- README.md | 17 +++++++---------- index.js | 41 +++++++++++++++++++++++++++++------------ test/index.js | 12 +++++------- 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 00f804c..c143807 100644 --- a/README.md +++ b/README.md @@ -17,22 +17,19 @@ npm install --save fs-db ``` var FsDb = require('fs-db') -var db = FsDb( - __dirname + '/things.csv', - 'csv' -) +var db = FsDb({ + location: __dirname + '/things.csv', + codec: 'csv' +}) db.readStream() .on('data', console.log) ``` -#### fsdown(location, options, codecOptions) - -`location` is the path to the database file. +#### fsdown(options) `options`: -- `codec` is which codec to use (defaults to 'csv'). can be a name of an existing codec or a custom codec object, see [codecs](./codecs) for what is expected of a codec. +- `location` is the path to the database file. +- `codec` is which codec to use (defaults to 'csv'). can be a name of an existing codec, a custom codec object (see [codecs](./codecs) for what is expected of a codec), or an array where the first item is one of the previous values and the second item is options to pass to the codec. - `keyAttribute` is a string identifier of the attribute used as keys (e.g. 'id'). - -`codecOptions` are passed to the codec. diff --git a/index.js b/index.js index 1f08271..2e8e4cc 100644 --- a/index.js +++ b/index.js @@ -13,23 +13,22 @@ var codecs = require('./codecs') module.exports = FsDb -function FsDb (location, options, codecOptions) { +function FsDb (options) { if (!(this instanceof FsDb)) - return new FsDb(location, options, codecOptions) + return new FsDb(options) - if (typeof location === 'undefined') { - throw new Error('fs-db: location is not defined') - } - - if (typeof options === 'string') { - options = { codec: options } + if (typeof options == 'string') { + options = { location: options } } else { options = defined(options, {}) } - this.location = location + if (options.location == null) { + throw new Error('fs-db: options.location is required.') + } + + this.location = options.location this.codec = getCodec(options.codec) - this.codecOptions = defined(codecOptions, {}) this.keyAttribute = defined(options.keyAttribute, 'key') } @@ -39,12 +38,30 @@ assign(FsDb.prototype, { }) function getCodec (codec) { + var codecOptions + if (Array.isArray(codec)) { + codec = codec[0] + codecOptions = codec[1] + } else { + codecOptions = {} + } if (typeof codec === 'string') { codec = codecs[codec] - } else if (codec == null || typeof codec !== 'object') { + } else if (!isCodec(codec)) { codec = codecs.csv } - return codec + return { + encode: codec.encode.bind(codec, codecOptions), + decode: codec.decode.bind(codec, codecOptions) + } +} + +function isCodec (codec) { + return ( + codec != null && + typeof codec.encode === 'function' && + typeof codec.decode === 'function' + ) } function createReadStream (options) { diff --git a/test/index.js b/test/index.js index 7d06ebe..96fdcc4 100644 --- a/test/index.js +++ b/test/index.js @@ -18,7 +18,7 @@ test('exports proper api', function (t) { }) test('csv .createReadStream()', function (t) { - var db = ctor('one.csv', 'csv') + var db = ctor({ location: 'one.csv', codec: 'csv' }) var readStream = db.createReadStream() toArray(readStream, function (err, data) { t.error(err, 'no error') @@ -29,7 +29,7 @@ test('csv .createReadStream()', function (t) { }) test('csv .createWriteStream()', function (t) { - var db = ctor('two.csv', 'csv') + var db = ctor({ location: 'two.csv', codec: 'csv' }) toStream(readData('one.json')) .pipe(db.createWriteStream()) .on('finish', function () { @@ -41,11 +41,9 @@ test('csv .createWriteStream()', function (t) { }) }) -function ctor (location, options, codecOptions) { - return FsDb( - Path.join(__dirname, 'data', location), - options, codecOptions - ) +function ctor (options) { + options.location = Path.join(__dirname, 'data', options.location) + return FsDb(options) } function readFile (file) { From 7d7483e0349ed53992e8598005c8352bad2d5c90 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Mon, 21 Sep 2015 21:30:00 +1200 Subject: [PATCH 16/18] fix debug --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 2e8e4cc..1c301f3 100644 --- a/index.js +++ b/index.js @@ -1,4 +1,4 @@ -var debug = require('debug')('fsdown') +var debug = require('debug')('fs-db') var fs = require('fs') var defined = require('defined') var inherits = require('inherits') From daaa9d790286332a971cb844ca63b72ea3988cb6 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Mon, 21 Sep 2015 21:30:29 +1200 Subject: [PATCH 17/18] fix order of ops --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 1c301f3..85a1254 100644 --- a/index.js +++ b/index.js @@ -40,8 +40,8 @@ assign(FsDb.prototype, { function getCodec (codec) { var codecOptions if (Array.isArray(codec)) { - codec = codec[0] codecOptions = codec[1] + codec = codec[0] } else { codecOptions = {} } From d9dbdb438ba863781afa9c1c6768c4ad00d37a17 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Mon, 21 Sep 2015 21:31:16 +1200 Subject: [PATCH 18/18] pumpify, formatting, and no more codecOptions since bound --- index.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 85a1254..6d17226 100644 --- a/index.js +++ b/index.js @@ -45,11 +45,13 @@ function getCodec (codec) { } else { codecOptions = {} } + if (typeof codec === 'string') { codec = codecs[codec] } else if (!isCodec(codec)) { codec = codecs.csv } + return { encode: codec.encode.bind(codec, codecOptions), decode: codec.decode.bind(codec, codecOptions) @@ -69,11 +71,12 @@ function createReadStream (options) { var keyAttribute = this.keyAttribute - // read from file - return fs.createReadStream(this.location) + return pumpify.obj([ + // read from file + fs.createReadStream(this.location), // parse data into objects - .pipe(this.codec.decode(this.codecOptions)) - .pipe(through.obj(function (row, enc, cb) { + this.codec.decode(), + through.obj(function (row, enc, cb) { // get key var key = row[keyAttribute] @@ -83,7 +86,8 @@ function createReadStream (options) { } cb(null, { key: key, value: row }) - })) + }) + ]) } function createWriteStream (options) { @@ -120,7 +124,7 @@ function createWriteStream (options) { } ), // format data to string - this.codec.encode(this.codecOptions), + this.codec.encode(), // write to file fs.createWriteStream(this.location) ])