diff --git a/README.md b/README.md index 1d17d91..c143807 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# fs-db +# fsdown -a silly **work in progress** to use the filesystem as a database. +a silly **work in progress** to use the filesystem (e.g. `.csv`, `.ldjson` files) as a database. the purpose is for apps to have human editable and readable data that can be iterated on easily (vim macros over sql migrations) and shared more openly (GitHub repos over JSON APIs). @@ -17,21 +17,19 @@ npm install --save fs-db ``` var FsDb = require('fs-db') -var fsDb = FsDb({ - location: __dirname + '/data', +var db = FsDb({ + location: __dirname + '/things.csv', + codec: 'csv' }) -fsDb.createReadStream() - .pipe(process.stdout) +db.readStream() + .on('data', console.log) ``` -#### FsDb(options) +#### fsdown(options) -possible `options` are: +`options`: -- `location`: root filesystem directory of the database -- `codec`: codec to use (defaults to 'json'), see [codecs](./codecs) - -#### fsDb.createReadStream() - -returns a readable [pull stream](https://npmjs.org/package/pull-stream) of objects with [JSON Pointer](https://npmjs.org/package/json-pointer) `id`s based on the path. +- `location` is the path to the database file. +- `codec` is which codec to use (defaults to 'csv'). can be a name of an existing codec, a custom codec object (see [codecs](./codecs) for what is expected of a codec), or an array where the first item is one of the previous values and the second item is options to pass to the codec. +- `keyAttribute` is a string identifier of the attribute used as keys (e.g. 'id'). diff --git a/codecs/csv.js b/codecs/csv.js index f84d7a0..33cf56d 100644 --- a/codecs/csv.js +++ b/codecs/csv.js @@ -1,11 +1,4 @@ -var csv = require('comma-separated-values') - module.exports = { - type: 'csv', - encode: function (obj) { - return csv.encode(obj, { header: true }) - }, - decode: function (str) { - return csv.parse(str, { header: true }) - } + decode: require('csv-parser'), + encode: require('csv-formatter') } diff --git a/codecs/index.js b/codecs/index.js index cfbc984..2f2a2ee 100644 --- a/codecs/index.js +++ b/codecs/index.js @@ -1,5 +1,3 @@ module.exports = { - json: require('./json'), - yml: require('./yml'), csv: require('./csv') } diff --git a/codecs/json.js b/codecs/json.js deleted file mode 100644 index ecc2f7a..0000000 --- a/codecs/json.js +++ /dev/null @@ -1,11 +0,0 @@ -var extend = require('xtend') -var encodings = require('level-codec/lib/encodings') - -module.exports = extend(encodings.json, { - encode: function encode (obj) { - return JSON.stringify(obj, null, 2) - }, - decode: function decode (str) { - return JSON.parse(str) - }, -}) diff --git a/codecs/yml.js b/codecs/yml.js deleted file mode 100644 index 5c13196..0000000 --- a/codecs/yml.js +++ /dev/null @@ -1,11 +0,0 @@ -var yaml = require('js-yaml') - -module.exports = { - type: 'yml', - encode: function (obj) { - return yaml.safeDump(obj) - }, - decode: function (str) { - return yaml.safeLoad(str) - } -} diff --git a/index.js b/index.js index f8a9bad..6d17226 100644 --- a/index.js +++ b/index.js @@ -1,34 +1,131 @@ -var pull = require('pull-stream') var debug = require('debug')('fs-db') +var fs = require('fs') +var defined = require('defined') +var inherits = require('inherits') +var assign = require('lodash.assign') +var forEach = require('lodash.foreach') +var through = require('through2') +var uuid = require('node-uuid') +var pumpify = require('pumpify') +var prepend = require('prepend-stream') var codecs = require('./codecs') module.exports = FsDb function FsDb (options) { - if (!(this instanceof FsDb)) { + if (!(this instanceof FsDb)) return new FsDb(options) + + if (typeof options == 'string') { + options = { location: options } + } else { + options = defined(options, {}) } - debug("constructor(", options, ")") - this.location = options.location || process.cwd() - this.fs = options.fs || require('fs') + if (options.location == null) { + throw new Error('fs-db: options.location is required.') + } - var codec = options.codec || 'json' - this.codec = (typeof codec === 'string') ? - codecs[codec] : codec + this.location = options.location + this.codec = getCodec(options.codec) + this.keyAttribute = defined(options.keyAttribute, 'key') } -FsDb.prototype = { +assign(FsDb.prototype, { createReadStream: createReadStream, + createWriteStream: createWriteStream +}) + +function getCodec (codec) { + var codecOptions + if (Array.isArray(codec)) { + codecOptions = codec[1] + codec = codec[0] + } else { + codecOptions = {} + } + + if (typeof codec === 'string') { + codec = codecs[codec] + } else if (!isCodec(codec)) { + codec = codecs.csv + } + + return { + encode: codec.encode.bind(codec, codecOptions), + decode: codec.decode.bind(codec, codecOptions) + } } -function createReadStream () { - debug('createReadStream()') - - return pull( - require('./lib/read-dir')(this), - require('./lib/read-file')(this), - require('./lib/parse')(this) +function isCodec (codec) { + return ( + codec != null && + typeof codec.encode === 'function' && + typeof codec.decode === 'function' ) } + +function createReadStream (options) { + debug('createReadStream(', options, ')') + + var keyAttribute = this.keyAttribute + + return pumpify.obj([ + // read from file + fs.createReadStream(this.location), + // parse data into objects + this.codec.decode(), + through.obj(function (row, enc, cb) { + // get key + var key = row[keyAttribute] + + // if no key, default to UUID + if (key == null) { + key = uuid() + } + + cb(null, { key: key, value: row }) + }) + ]) +} + +function createWriteStream (options) { + debug('createWriteStream(', options, ')') + + var table = {} + + return pumpify.obj([ + // parse values as json + // add current data to beginning of + // the data that is to be written + prepend.obj( + this.createReadStream(options) + ), + // construct in-memory table of data + through.obj( + function transform (row, enc, cb) { + debug("transform row", row) + // perform operation to table + if (row.type === 'del') { + delete table[row.key] + } else { + table[row.key] = row.value + } + cb() + }, + function flush (cb) { + // output contents of table + debug("flush table", table) + forEach(table, function (value, key) { + this.push(value) + }, this) + cb() + } + ), + // format data to string + this.codec.encode(), + // write to file + fs.createWriteStream(this.location) + ]) +} diff --git a/lib/parse.js b/lib/parse.js deleted file mode 100644 index a588e34..0000000 --- a/lib/parse.js +++ /dev/null @@ -1,68 +0,0 @@ -var isPlainObject = require('is-plain-object') -var toJsonPointer = require('json-pointer').compile -var pull = require('pull-stream') -var debug = require('debug')('fs-db:parse') - -module.exports = pull.Through(function contentParser (read, options) { - - var codec = options.codec - var ended = false - - return function parseContent (end, cb) { - read(end, function (end, file) { - if (end) { - // HACK: not sure why `ended` is necessary - if (ended) { return } - ended = true - return process.nextTick(function () { - cb(end) - }) - } - debug("file", file) - - var rootPath = file.path.split('.' + codec.type)[0] - - // TODO try catch - var content = codec.decode(file.content) - - debug("content", content) - - // identify nodes in contentObj - traverse(content, [], function (obj, path) { - - if (!obj.id && path.length === 0) { - obj.id = rootPath - } else if (!obj.id) { - obj.id = rootPath + "#" + toJsonPointer(path) - } - - debug("pushing", obj) - cb(null, obj) - - return obj.id - }) - }) - } -}) - -function traverse (obj, path, cb) { - debug("traverse", obj, path) - - if (!isPlainObject(obj)) { - return obj - } - - Object.keys(obj).forEach(function (key) { - var val = obj[key] - - if (isPlainObject(val)) { - obj[key] = traverse(val, path.concat(key), cb) - } else if (Array.isArray(val)) { - obj[key] = val.map(function (item, index) { - return traverse(item, path.concat([key, index]), cb) - }) - } - }) - - return cb(obj, path) -} diff --git a/lib/read-dir.js b/lib/read-dir.js deleted file mode 100644 index eeea1bb..0000000 --- a/lib/read-dir.js +++ /dev/null @@ -1,15 +0,0 @@ -var readdirp = require('readdirp') -var streamToPull = require('stream-to-pull-stream') -var debug = require('debug')('fs-db:read-dir') - -module.exports = function readDir (options) { - - var readdirpOptions = { - root: options.location, - fileFilter: '*.' + options.codec.type, - } - debug("readdirp", readdirpOptions) - return streamToPull.source( - readdirp(readdirpOptions) - ) -} diff --git a/lib/read-file.js b/lib/read-file.js deleted file mode 100644 index a9a9386..0000000 --- a/lib/read-file.js +++ /dev/null @@ -1,27 +0,0 @@ -var paraMap = require('pull-paramap') -var debug = require('debug')('fs-db') - -module.exports = function contentReader (options) { - - var fs = options.fs - - return paraMap(readContent) - - function readContent (entry, cb) { - debug("entry", entry) - - debug('readFile(', entry.fullPath, ')') - fs.readFile(entry.fullPath, 'utf8', function (err, content) { - debug('readFile() ->', err, content) - - if (err) { return cb(err) } - - var file = { - path: entry.path, - content: content, - } - debug('pushing', file) - cb(null, file) - }) - } -} diff --git a/package.json b/package.json index 7719606..226410b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fs-db", - "version": "2.1.3", + "version": "3.0.0-pre", "description": "wip to use fs as a db", "main": "index.js", "scripts": { @@ -8,43 +8,43 @@ }, "repository": { "type": "git", - "url": "https://github.com/ahdinosaur/fs-db.git" + "url": "https://github.com/holodex/fs-db.git" }, "keywords": [ "directory", - "analyze", - "walk", "json", - "yaml", - "model", + "csv", + "tsv", "fs", + "model", + "collection", "database", - "human", - "readable" + "human" ], "author": "ahdinosaur", "license": "ISC", "bugs": { - "url": "https://github.com/ahdinosaur/fs-db/issues" + "url": "https://github.com/holodex/fs-db/issues" }, - "homepage": "https://github.com/ahdinosaur/fs-db", + "homepage": "https://github.com/holodex/fs-db", "devDependencies": { - "pull-array-collate": "^0.1.0", - "sort-by": "^1.1.0", - "sort-keys": "^1.0.0", - "tape": "^3.5.0" + "sort-by": "^1.1.1", + "sort-keys": "^1.1.1", + "stream-array": "^1.1.1", + "stream-to-array": "^2.0.2", + "tape": "^4.2.0" }, "dependencies": { - "comma-separated-values": "^3.6.2", - "debug": "^2.1.3", - "is-plain-object": "^1.0.0", - "js-yaml": "^3.3.1", - "json-pointer": "^0.3.0", - "level-codec": "^4.2.0", - "pull-paramap": "^1.1.1", - "pull-stream": "^2.26.0", - "readdirp": "^1.3.0", - "stream-to-pull-stream": "^1.6.1", - "xtend": "^4.0.0" + "csv-formatter": "^1.0.0", + "csv-parser": "^1.7.0", + "debug": "^2.2.0", + "defined": "^1.0.0", + "inherits": "^2.0.1", + "lodash.assign": "^3.2.0", + "lodash.foreach": "^3.0.3", + "node-uuid": "^1.4.3", + "prepend-stream": "holodex/prepend-stream", + "pumpify": "^1.0.2", + "through2": "^2.0.0" } } diff --git a/test/data.json b/test/data.json deleted file mode 100644 index 9d2dbf7..0000000 --- a/test/data.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "10": [ - "11", - "12", - "abc/stu#/10/2" - ], - "id": "abc/stu" - }, - { - "13": "14", - "id": "abc/stu#/10/2" - }, - { - "0": "1", - "id": "def/ghi/jkl/mno" - }, - { - "2": "3", - "4": "def/pqr#/4", - "id": "def/pqr" - }, - { - "5": "6", - "7": "def/pqr#/4/7", - "id": "def/pqr#/4" - }, - { - "8": "9", - "id": "def/pqr#/4/7" - } -] diff --git a/test/data/abc/stu.json b/test/data/abc/stu.json deleted file mode 100644 index 554026b..0000000 --- a/test/data/abc/stu.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "10": [ - "11", - "12", - { - "13": "14" - } - ] -} diff --git a/test/data/def/ghi/jkl/mno.json b/test/data/def/ghi/jkl/mno.json deleted file mode 100644 index 5af0f90..0000000 --- a/test/data/def/ghi/jkl/mno.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "0": "1" -} diff --git a/test/data/def/pqr.json b/test/data/def/pqr.json deleted file mode 100644 index 575979d..0000000 --- a/test/data/def/pqr.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "2": "3", - "4": { - "5": "6", - "7": { - "8": "9" - } - } -} diff --git a/test/data/one.csv b/test/data/one.csv new file mode 100644 index 0000000..bf9947e --- /dev/null +++ b/test/data/one.csv @@ -0,0 +1,4 @@ +key,name,description +0,bed,a comfy couch +1,chair,a sweet spot +2,desk,a clean surface diff --git a/test/data/one.json b/test/data/one.json new file mode 100644 index 0000000..7d86ae8 --- /dev/null +++ b/test/data/one.json @@ -0,0 +1,26 @@ +[ + { + "key": "0", + "value": { + "key": "0", + "name": "bed", + "description": "a comfy couch" + } + }, + { + "key": "1", + "value": { + "key": "1", + "name": "chair", + "description": "a sweet spot" + } + }, + { + "key": "2", + "value": { + "key": "2", + "name": "desk", + "description": "a clean surface" + } + } +] diff --git a/test/index.js b/test/index.js index 1083b4d..96fdcc4 100644 --- a/test/index.js +++ b/test/index.js @@ -1,55 +1,61 @@ var test = require('tape') var fs = require('fs') var Path = require('path') -var extend = require('xtend') +var toStream = require('stream-array') +var toArray = require('stream-to-array') var sortKeys = require('sort-keys') var sortBy = require('sort-by') -var pull = require('pull-stream') -var pullToArray = require('pull-array-collate') -var streamToArray = require('stream-to-array') var FsDb = require('../') -test('Constructor', function (t) { +test('exports proper api', function (t) { t.equal(typeof FsDb, 'function') - var fsDb = ctor() - t.ok(fsDb) - t.equal(typeof fsDb, 'object') - t.equal(typeof fsDb.createReadStream, 'function') + var db = FsDb('data/one.csv') + t.equal(typeof db, 'object') + t.equal(typeof db.createReadStream, 'function') + t.equal(typeof db.createWriteStream, 'function') t.end() }) -test('.createReadStream()', function (t) { - var fsDb = ctor() - var readStream = fsDb.createReadStream() - pull( - readStream, - pullToArray(), - pull.drain(function (data) { - var expected = readData('./data.json') - var actual = sortData(data) - t.deepEqual(actual, expected) - }, function (err) { - t.notOk(err, 'no error') +test('csv .createReadStream()', function (t) { + var db = ctor({ location: 'one.csv', codec: 'csv' }) + var readStream = db.createReadStream() + toArray(readStream, function (err, data) { + t.error(err, 'no error') + var expected = readData('one.json') + t.deepEqual(data, expected, 'data is correct') + t.end() + }) +}) + +test('csv .createWriteStream()', function (t) { + var db = ctor({ location: 'two.csv', codec: 'csv' }) + toStream(readData('one.json')) + .pipe(db.createWriteStream()) + .on('finish', function () { + t.deepEqual( + readFile('two.csv'), + readFile('one.csv') + ) t.end() }) - ) }) function ctor (options) { - return FsDb(extend({ - location: __dirname + '/data', - }, options || {})) + options.location = Path.join(__dirname, 'data', options.location) + return FsDb(options) } -function readData (file) { - return JSON.parse( - fs.readFileSync( - Path.join(__dirname, file), 'utf8' - ) +function readFile (file) { + return fs.readFileSync( + Path.join(__dirname, 'data', file), 'utf8' ) } +function readData (file) { + return JSON.parse(readFile(file)) +} + function sortData (data) { return data.map(function (item) { return sortKeys(item)