From c2b1520627eb1d6b55766d2a411c7b1e622e6622 Mon Sep 17 00:00:00 2001 From: Tuomas Peippo Date: Tue, 14 Apr 2020 14:08:51 +0300 Subject: [PATCH 01/12] Add initial transactWriteItem implementation --- actions/transactWriteItems.js | 89 +++++++++++++++++++ index.js | 2 +- validations/transactWriteItems.js | 138 ++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 actions/transactWriteItems.js create mode 100644 validations/transactWriteItems.js diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js new file mode 100644 index 0000000..06b27da --- /dev/null +++ b/actions/transactWriteItems.js @@ -0,0 +1,89 @@ +var async = require('async'), + putItem = require('./putItem'), + deleteItem = require('./deleteItem'), + updateItem = require('./updateItem'), + db = require('../db') + +module.exports = function transactWriteItem(store, data, cb) { + var actions = [] + + async.series([ + async.eachSeries.bind(async, data.TransactItems, addActions), + async.series.bind(async, actions), + ], function(err, responses) { + if (err) { + if (err.body && (/Missing the key/.test(err.body.message) || /Type mismatch for key/.test(err.body.message))) + err.body.message = 'The provided key element does not match the schema' + return cb(err) + } + var res = {UnprocessedItems: {}}, tableUnits = {} + + if (~['TOTAL', 'INDEXES'].indexOf(data.ReturnConsumedCapacity)) { + responses[1].forEach(function(action) { + var table = action.ConsumedCapacity.TableName + if (!tableUnits[table]) tableUnits[table] = 0 + tableUnits[table] += action.ConsumedCapacity.CapacityUnits + }) + res.ConsumedCapacity = Object.keys(tableUnits).map(function(table) { + return { + CapacityUnits: tableUnits[table], + TableName: table, + Table: data.ReturnConsumedCapacity == 'INDEXES' ? {CapacityUnits: tableUnits[table]} : undefined, + } + }) + } + + cb(null, res) + }) + + function addActions(transactItem, cb) { + var options = {} + var tableName + + if (data.ReturnConsumedCapacity) options.ReturnConsumedCapacity = data.ReturnConsumedCapacity + + if (transactItem.Put) { + tableName = transactItem.Put.TableName; + options = {TableName: tableName} + + store.getTable(tableName, function(err, table) { + if (err) return cb(err) + if ((err = db.validateItem(transactItem.Put.Item, table)) != null) return cb(err) + + options.Item = transactItem.Put.Item + actions.push(putItem.bind(null, store, options)) + db.createKey(options.Item, table) + + cb() + }) + } else if (transactItem.Delete) { + tableName = transactItem.Delete.TableName; + options = {TableName: tableName} + + store.getTable(tableName, function(err, table) { + if (err) return cb(err) + if ((err = db.validateKey(transactItem.Delete.Key, table) != null)) return cb(err) + + options.Key = transactItem.Delete.Key + actions.push(deleteItem.bind(null, store, options)) + + db.createKey(options.Key, table) + + cb() + }) + } else if (transactItem.Update) { + + store.getTable(tableName, function(err, table) { + if (err) return cb(err) + if ((err = db.validateKey(transactItem.Update.Key, table) != null)) return cb(err) + + options.Key = transactItem.Update.Key + actions.push(updateItem.bind(null, store, options)) + + db.createKey(options.Key, table) + + cb() + }) + } + } +} diff --git a/index.js b/index.js index 1c169ca..6a7385d 100644 --- a/index.js +++ b/index.js @@ -13,7 +13,7 @@ var MAX_REQUEST_BYTES = 16 * 1024 * 1024 var validApis = ['DynamoDB_20111205', 'DynamoDB_20120810'], validOperations = ['BatchGetItem', 'BatchWriteItem', 'CreateTable', 'DeleteItem', 'DeleteTable', 'DescribeTable', 'DescribeTimeToLive', 'GetItem', 'ListTables', 'PutItem', 'Query', 'Scan', 'TagResource', - 'UntagResource', 'ListTagsOfResource', 'UpdateItem', 'UpdateTable'], + 'UntagResource', 'ListTagsOfResource', 'UpdateItem', 'UpdateTable', 'TransactWriteItems'], actions = {}, actionValidations = {} diff --git a/validations/transactWriteItems.js b/validations/transactWriteItems.js new file mode 100644 index 0000000..6126ab9 --- /dev/null +++ b/validations/transactWriteItems.js @@ -0,0 +1,138 @@ +var validations = require('./index'), + db = require('../db') + +exports.types = { + ReturnConsumedCapacity: { + type: 'String', + enum: ['INDEXES', 'TOTAL', 'NONE'], + }, + ReturnItemCollectionMetrics: { + type: 'String', + enum: ['SIZE', 'NONE'], + }, + ClientRequestToken: { + type: 'String' + }, + TransactItems: { + type: 'List', + notNull: true, + lengthGreaterThanOrEqual: 1, + children: { + type: 'ValueStruct', + children: { + Put: { + type: 'FieldStruct', + children: { + TableName: { + type: 'String', + }, + ExpressionAttributeValues: { + type: 'Map', + children: 'AttrStruct', + }, + ExpressionAttributeNames: { + type: 'Map', + children: 'String', + }, + ReturnValuesOnConditionCheckFailure: { + type: 'String', + enum: ['ALL_OLD', 'NONE'], + }, + ConditionExpression: { + type: 'String', + }, + Item: { + type: 'Map', + notNull: true, + children: 'AttrStruct', + }, + }, + }, + Update: { + type: 'FieldStruct', + children: { + TableName: { + type: 'String', + }, + ExpressionAttributeValues: { + type: 'Map', + children: 'AttrStruct', + }, + ExpressionAttributeNames: { + type: 'Map', + children: 'String', + }, + ReturnValuesOnConditionCheckFailure: { + type: 'String', + enum: ['ALL_OLD', 'NONE'], + }, + ConditionExpression: { + type: 'String', + }, + Key: { + type: 'Map', + notNull: true, + children: 'AttrStruct', + }, + }, + }, + Delete: { + type: 'FieldStruct', + children: { + TableName: { + type: 'String', + }, + ExpressionAttributeValues: { + type: 'Map', + children: 'AttrStruct', + }, + ExpressionAttributeNames: { + type: 'Map', + children: 'String', + }, + ReturnValuesOnConditionCheckFailure: { + type: 'String', + enum: ['ALL_OLD', 'NONE'], + }, + ConditionExpression: { + type: 'String', + }, + Key: { + type: 'Map', + notNull: true, + children: 'AttrStruct', + }, + }, + }, + ConditionCheck: { + type: 'FieldStruct', + children: { + TableName: { + type: 'String', + }, + ExpressionAttributeValues: { + type: 'Map', + children: 'AttrStruct', + }, + ExpressionAttributeNames: { + type: 'Map', + children: 'String', + }, + ReturnValuesOnConditionCheckFailure: { + type: 'String', + enum: ['ALL_OLD', 'NONE'], + }, + ConditionExpression: { + type: 'String', + }, + Key: { + type: 'Map', + notNull: true, + children: 'AttrStruct', + }, + }, + }, + }, + }, + }, +} From 6caa634ccd5083796839bb77fe6d6cc547e78d89 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Tue, 27 Jul 2021 15:29:19 -0400 Subject: [PATCH 02/12] start working on TransactWriteItem tests --- actions/transactWriteItems.js | 137 +++--- db/index.js | 11 + test/helpers.js | 18 + test/transactWriteItem.js | 693 ++++++++++++++++++++++++++++++ validations/transactWriteItems.js | 35 ++ 5 files changed, 836 insertions(+), 58 deletions(-) create mode 100644 test/transactWriteItem.js diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index 06b27da..b0ed9e5 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -5,85 +5,106 @@ var async = require('async'), db = require('../db') module.exports = function transactWriteItem(store, data, cb) { - var actions = [] - - async.series([ - async.eachSeries.bind(async, data.TransactItems, addActions), - async.series.bind(async, actions), - ], function(err, responses) { - if (err) { - if (err.body && (/Missing the key/.test(err.body.message) || /Type mismatch for key/.test(err.body.message))) - err.body.message = 'The provided key element does not match the schema' - return cb(err) - } - var res = {UnprocessedItems: {}}, tableUnits = {} - - if (~['TOTAL', 'INDEXES'].indexOf(data.ReturnConsumedCapacity)) { - responses[1].forEach(function(action) { - var table = action.ConsumedCapacity.TableName - if (!tableUnits[table]) tableUnits[table] = 0 - tableUnits[table] += action.ConsumedCapacity.CapacityUnits - }) - res.ConsumedCapacity = Object.keys(tableUnits).map(function(table) { - return { - CapacityUnits: tableUnits[table], - TableName: table, - Table: data.ReturnConsumedCapacity == 'INDEXES' ? {CapacityUnits: tableUnits[table]} : undefined, + var actions = [] + var seenKeys = {} + + async.series([ + async.eachSeries.bind(async, data.TransactItems, addActions), + async.series.bind(async, actions), + ], function (err, responses) { + console.log('wake up at 11:30') + console.dir(err) + console.dir(responses) + + if (err) { + if (err.body && (/Missing the key/.test(err.body.message) || /Type mismatch for key/.test(err.body.message))) + err.body.message = 'The provided key element does not match the schema' + return cb(err) } - }) - } + var res = {UnprocessedItems: {}}, tableUnits = {} - cb(null, res) - }) + if (~['TOTAL', 'INDEXES'].indexOf(data.ReturnConsumedCapacity)) { + responses[1].forEach(function (action) { + var table = action.ConsumedCapacity.TableName + if (!tableUnits[table]) tableUnits[table] = 0 + tableUnits[table] += action.ConsumedCapacity.CapacityUnits + }) + res.ConsumedCapacity = Object.keys(tableUnits).map(function (table) { + return { + CapacityUnits: tableUnits[table], + TableName: table, + Table: data.ReturnConsumedCapacity == 'INDEXES' ? {CapacityUnits: tableUnits[table]} : undefined, + } + }) + } - function addActions(transactItem, cb) { - var options = {} - var tableName + cb(null, res) + }) - if (data.ReturnConsumedCapacity) options.ReturnConsumedCapacity = data.ReturnConsumedCapacity + function addActions(transactItem, cb) { + var options = {} + var tableName + var key - if (transactItem.Put) { + if (data.ReturnConsumedCapacity) options.ReturnConsumedCapacity = data.ReturnConsumedCapacity + + if (transactItem.Put) { tableName = transactItem.Put.TableName; options = {TableName: tableName} - store.getTable(tableName, function(err, table) { - if (err) return cb(err) - if ((err = db.validateItem(transactItem.Put.Item, table)) != null) return cb(err) + store.getTable(tableName, function (err, table) { + if (err) return cb(err) + if ((err = db.validateItem(transactItem.Put.Item, table)) != null) return cb(err) - options.Item = transactItem.Put.Item - actions.push(putItem.bind(null, store, options)) - db.createKey(options.Item, table) + options.Item = transactItem.Put.Item + actions.push(putItem.bind(null, store, options)) + key = db.createKey(options.Item, table) - cb() + if (seenKeys[key]) { + return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) + } + seenKeys[key] = true + return cb() }) - } else if (transactItem.Delete) { + } else if (transactItem.Delete) { tableName = transactItem.Delete.TableName; options = {TableName: tableName} - store.getTable(tableName, function(err, table) { - if (err) return cb(err) - if ((err = db.validateKey(transactItem.Delete.Key, table) != null)) return cb(err) + store.getTable(tableName, function (err, table) { + if (err) return cb(err) + if ((err = db.validateKey(transactItem.Delete.Key, table) != null)) return cb(err) - options.Key = transactItem.Delete.Key - actions.push(deleteItem.bind(null, store, options)) + options.Key = transactItem.Delete.Key + actions.push(deleteItem.bind(null, store, options)) - db.createKey(options.Key, table) + key = db.createKey(options.Key, table) - cb() + if (seenKeys[key]) { + return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) + } + seenKeys[key] = true + return cb() }) - } else if (transactItem.Update) { + } else if (transactItem.Update) { + tableName = transactItem.Update.TableName; + options = transactItem.Update - store.getTable(tableName, function(err, table) { - if (err) return cb(err) - if ((err = db.validateKey(transactItem.Update.Key, table) != null)) return cb(err) + store.getTable(tableName, function (err, table) { + if (err) return cb(err) + if ((err = db.validateKey(transactItem.Update.Key, table) != null)) return cb(err) - options.Key = transactItem.Update.Key - actions.push(updateItem.bind(null, store, options)) + options.Key = transactItem.Update.Key + actions.push(updateItem.bind(null, store, options)) - db.createKey(options.Key, table) + key = db.createKey(options.Key, table) - cb() + if (seenKeys[key]) { + return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) + } + seenKeys[key] = true + return cb() }) - } - } + } + + } } diff --git a/db/index.js b/db/index.js index 03f29d4..8bc13a8 100644 --- a/db/index.js +++ b/db/index.js @@ -25,6 +25,7 @@ exports.toRangeStr = toRangeStr exports.toLexiStr = toLexiStr exports.hashPrefix = hashPrefix exports.validationError = validationError +exports.transactionCancelledException = transactionCancelledException exports.limitError = limitError exports.checkConditional = checkConditional exports.itemSize = itemSize @@ -478,6 +479,16 @@ function limitError(msg) { return err } +function transactionCancelledException(msg) { + var err = new Error(msg) + err.statusCode = 400 + err.body = { + __type: 'com.amazonaws.dynamodb.v20120810#TransactionCanceledException', + message: msg + } + return err +} + function itemSize(item, compress, addMetaSize, rangeKey) { // Size of compressed item (for checking query/scan limit) seems complicated, // probably due to some internal serialization format. diff --git a/test/helpers.js b/test/helpers.js index 9e10b89..b6ab43e 100644 --- a/test/helpers.js +++ b/test/helpers.js @@ -25,6 +25,7 @@ exports.batchBulkPut = batchBulkPut exports.assertSerialization = assertSerialization exports.assertType = assertType exports.assertValidation = assertValidation +exports.assertTransactionCanceled = assertTransactionCanceled exports.assertNotFound = assertNotFound exports.assertInUse = assertInUse exports.assertConditional = assertConditional @@ -578,6 +579,23 @@ function assertValidation(target, data, msg, done) { }) } +function assertTransactionCanceled(target, data, msg, done) { + request(opts(target, data), function(err, res) { + if (err) return done(err) + if (typeof res.body !== 'object') { + return done(new Error('Not JSON: ' + res.body)) + } + res.body.__type.should.equal('com.amazonaws.dynamodb.v20120810#TransactionCanceledException') + if (msg instanceof RegExp) { + res.body.message.should.match(msg) + } else { + res.body.message.should.equal(msg) + } + res.statusCode.should.equal(400) + done() + }) +} + function assertNotFound(target, data, msg, done) { request(opts(target, data), function(err, res) { if (err) return done(err) diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js new file mode 100644 index 0000000..dfeb11a --- /dev/null +++ b/test/transactWriteItem.js @@ -0,0 +1,693 @@ +var async = require('async'), + helpers = require('./helpers'), + db = require('../db') + +var target = 'TransactWriteItems', + request = helpers.request, + randomName = helpers.randomName, + opts = helpers.opts.bind(null, target), + assertType = helpers.assertType.bind(null, target), + assertValidation = helpers.assertValidation.bind(null, target), + assertTransactionCanceled = helpers.assertTransactionCanceled.bind(null, target) + assertNotFound = helpers.assertNotFound.bind(null, target) + +describe('transactWriteItem', function() { + + describe('serializations', function() { + + it('should return SerializationException when TransactItems is not a list', function(done) { + assertType('TransactItems', 'List', done) + }) + + it('should return SerializationException when TransactItems.0.Delete.Key is not a map', function(done) { + assertType('TransactItems.0.Delete.Key', 'Map', done) + }) + + it('should return SerializationException when TransactItems.0.Delete.Key.Attr is not an attr struct', function(done) { + this.timeout(60000) + assertType('TransactItems.0.Delete.Key.Attr', 'AttrStruct', done) + }) + + it('should return SerializationException when TransactItems.0.Put is not a struct', function(done) { + assertType('TransactItems.0.Put', 'FieldStruct', done) + }) + + it('should return SerializationException when TransactItems.0.Put.Item is not a map', function(done) { + assertType('TransactItems.0.Put.Item', 'Map', done) + }) + + it('should return SerializationException when TransactItems.0.Put.Item.Attr is not an attr struct', function(done) { + this.timeout(60000) + assertType('TransactItems.0.Put.Item.Attr', 'AttrStruct', done) + }) + + it('should return SerializationException when TransactItems.0.Update is not a struct', function(done) { + assertType('TransactItems.0.Update', 'FieldStruct', done) + }) + + it('should return SerializationException when TransactItems.0.Update.UpdateExpression is not a string', function(done) { + assertType('TransactItems.0.Update.UpdateExpression', 'String', done) + }) + + it('should return SerializationException when ReturnConsumedCapacity is not a string', function(done) { + assertType('ReturnConsumedCapacity', 'String', done) + }) + + it('should return SerializationException when ReturnItemCollectionMetrics is not a string', function(done) { + assertType('ReturnItemCollectionMetrics', 'String', done) + }) + }) + + describe('validations', function() { + + it('should return ValidationException for empty body', function (done) { + assertValidation({}, + '1 validation error detected: ' + + 'Value null at \'transactItems\' failed to satisfy constraint: ' + + 'Member must not be null', done) + }) + + it('should return ValidationException for missing TransactItems', function (done) { + assertValidation({ReturnConsumedCapacity: 'hi', ReturnItemCollectionMetrics: 'hi'}, [ + 'Value \'hi\' at \'returnConsumedCapacity\' failed to satisfy constraint: ' + + 'Member must satisfy enum value set: [INDEXES, TOTAL, NONE]', + 'Value \'hi\' at \'returnItemCollectionMetrics\' failed to satisfy constraint: ' + + 'Member must satisfy enum value set: [SIZE, NONE]', + 'Value null at \'transactItems\' failed to satisfy constraint: ' + + 'Member must not be null', + ], done) + }) + + it('should return ValidationException for empty TransactItems', function (done) { + assertValidation({TransactItems: []}, + '1 validation error detected: ' + + 'Value \'[]\' at \'transactItems\' failed to satisfy constraint: ' + + 'Member must have length greater than or equal to 1', done) + }) + + it('should return ValidationException for invalid update request in TransactItems', function (done) { + assertValidation({TransactItems: [{Update: {}}]}, + '1 validation error detected: ' + + 'Value null at \'transactItems.1.member.update.key\' failed to satisfy constraint: ' + + 'Member must not be null', done) + }) + + + it('should return ValidationException for invalid put request in TransactItems', function (done) { + assertValidation({TransactItems: [{Put: {}}]}, + '1 validation error detected: ' + + 'Value null at \'transactItems.1.member.put.item\' failed to satisfy constraint: ' + + 'Member must not be null', done) + }) + + + it('should return ValidationException for invalid delete request in TransactItems', function (done) { + assertValidation({TransactItems: [{Delete: {}}]}, + '1 validation error detected: ' + + 'Value null at \'transactItems.1.member.delete.key\' failed to satisfy constraint: ' + + 'Member must not be null', done) + }) + + it('should return ValidationException for invalid metadata and missing requests', function (done) { + assertValidation({TransactItems: [], ReturnConsumedCapacity: 'hi', ReturnItemCollectionMetrics: 'hi'}, [ + 'Value \'hi\' at \'returnConsumedCapacity\' failed to satisfy constraint: ' + + 'Member must satisfy enum value set: [INDEXES, TOTAL, NONE]', + 'Value \'hi\' at \'returnItemCollectionMetrics\' failed to satisfy constraint: ' + + 'Member must satisfy enum value set: [SIZE, NONE]', + 'Value \'[]\' at \'transactItems\' failed to satisfy constraint: ' + + 'Member must have length greater than or equal to 1', + ], done) + }) + + it('should return ValidationException for incorrect attributes', function (done) { + assertValidation({ + TransactItems: [{Put: {}, Delete: {}}], + ReturnConsumedCapacity: 'hi', ReturnItemCollectionMetrics: 'hi' + }, [ + 'Value \'hi\' at \'returnConsumedCapacity\' failed to satisfy constraint: ' + + 'Member must satisfy enum value set: [INDEXES, TOTAL, NONE]', + 'Value \'hi\' at \'returnItemCollectionMetrics\' failed to satisfy constraint: ' + + 'Member must satisfy enum value set: [SIZE, NONE]', + 'Value null at \'transactItems.1.member.delete.key\' failed to satisfy constraint: ' + + 'Member must not be null', + 'Value null at \'transactItems.1.member.put.item\' failed to satisfy constraint: ' + + 'Member must not be null', + ], done) + }) + + it('should return ValidationException when writing more than 25 items', function (done) { + var requests = [], i + for (i = 0; i < 26; i++) { + requests.push(i % 2 ? {Delete: {Key: {a: {S: String(i)}}}} : {Put: {Item: {a: {S: String(i)}}}}) + } + assertValidation({TransactItems: requests}, + [new RegExp('Member must have length less than or equal to 25')], done) + }) + + it('should return ResourceNotFoundException when fetching exactly 25 items and table does not exist', function (done) { + var requests = [], i + for (i = 0; i < 25; i++) { + requests.push(i % 2 ? {Delete: {TableName: 'a', Key: {a: {S: String(i)}}}} : { + Put: { + TableName: 'a', + Item: {a: {S: String(i)}} + } + }) + } + assertNotFound({TransactItems: requests}, + 'Requested resource not found', done) + }) + + it('should check table exists first before checking for duplicate keys', function (done) { + assertNotFound({ + TransactItems: [{Delete: {TableName: 'a', Key: {a: {S: '1'}}}}, { + Put: { + TableName: 'a', + Item: {a: {S: '1'}} + } + }] + }, + 'Requested resource not found', done) + }) + + it('should return TransactionCanceledException for puts and deletes of the same item with delete first', function (done) { + var transaction = { + TransactItems: [{ + Delete: { + TableName: helpers.testHashTable, + Key: {a: {S: 'aaaaa'}} + } + }, {Put: {TableName: helpers.testHashTable, Item: {a: {S: 'aaaaa'}}}}] + } + assertTransactionCanceled(transaction, 'Transaction cancelled, please refer cancellation reasons for specific reasons', done) + }) + + it('should return TransactionCanceledException for puts and deletes of the same item with put first', function (done) { + var transaction = { + TransactItems: [{ + Put: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }, {Delete: {TableName: helpers.testHashTable, Key: {a: {S: 'aaaaa'}}}}] + } + assertTransactionCanceled(transaction, 'Transaction cancelled, please refer cancellation reasons for specific reasons', done) + }) + + it('should return ValidationException for key type mismatch in Put Item', function (done) { + async.forEach([ + {NULL: true}, + {M: {a: {S: ''}}}, + {L: [{M: {a: {S: ''}}}]} + ], function (expr, cb) { + assertValidation({TransactItems: [{Put: {TableName: helpers.testHashTable, Item: {a: expr}}}]}, + 'The provided key element does not match the schema', cb) + }, done) + }) + + it('should return ValidationException for single invalid action', function (done) { + var transaction = { + TransactItems: [{ + NotARealAction: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }] + } + assertValidation(transaction, 'The action or operation requested is invalid. Verify that the action is typed correctly.', done) + }) + + it('should return ValidationException for one invalid action and one valid action', function (done) { + var transaction = { + TransactItems: [{ + NotARealAction: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }, + { + Put: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }] + } + assertValidation(transaction, 'The action or operation requested is invalid. Verify that the action is typed correctly.', done) + }) + + + it('should return ValidationException for multiple invalid actions', function (done) { + var transaction = { + TransactItems: [{ + NotARealAction: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }, + { + AnotherFakeAction: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }] + } + assertValidation(transaction, 'The action or operation requested is invalid. Verify that the action is typed correctly.', done) + }) + + describe('functionality', function() { + it('should write a single item', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + batchReq = {TransactItems: []} + batchReq.TransactItems = [{Put: {TableName: helpers.testHashTable, Item: item}}] + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({Item: item}) + done() + }) + }) + }) + + it('should write multiple items', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + transactReq.TransactItems = [{Put: {TableName: helpers.testHashTable, Item: item}}, {Put: {TableName: helpers.testHashTable, Item: item2}}] + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({Item: item}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({Item: item2}) + done() + }) + }) + }) + }) + + it('should write, update, and delete in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item3 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: item3 + } + }, + { + Update: { + TableName: helpers.testHashTable, + Key: { + a: item.a + }, + UpdateExpression: 'SET c=:d', + ExpressionAttributeValues: { + ':d': { + S: 'd' + } + } + } + }, + { + Delete: { + TableName: helpers.testHashTable, + Key: { + a: item2.a + } + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // update item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.eql({...item, c: {S: 'd'}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item3.a}, ConsistentRead: true}), function(err, res) { + // put item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({Item: item3}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { + // delete item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({}) + done() + }) + }) + }) + }) + }) + }) + }) + + it('should write & update with condition expression in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: item + } + }, + { + Update: { + TableName: helpers.testHashTable, + Key: { + a: item2.a + }, + ConditionExpression: 'attribute_not_exists(f)', + UpdateExpression: 'SET c=:d', + ExpressionAttributeValues: { + ':d': { + S: 'd' + } + } + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // update item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.eql(item) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { + // put item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.eql({...item2, c: {S: 'd'}}) + done() + }) + }) + }) + }) + }) + + it('should fail to write & update with failed condition expression in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: item + } + }, + { + Update: { + TableName: helpers.testHashTable, + Key: { + a: item2.a + }, + ConditionExpression: 'attribute_not_exists(c)', + UpdateExpression: 'SET c=:d', + ExpressionAttributeValues: { + ':d': { + S: 'd' + } + } + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(400) + res.body.message.should.equal('The conditional request failed') + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // update item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.equal({}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { + // put item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.eql(item2) + done() + }) + }) + }) + }) + }) + // + // it('should delete an item from each table', function(done) { + // var item = {a: {S: helpers.randomString()}, c: {S: 'c'}}, + // item2 = {a: {S: helpers.randomString()}, b: {S: helpers.randomString()}, c: {S: 'c'}}, + // batchReq = {TransactItems: {}} + // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}] + // batchReq.TransactItems[helpers.testRangeTable] = [{DeleteRequest: {Key: {a: item2.a, b: item2.b}}}] + // request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // request(helpers.opts('PutItem', {TableName: helpers.testRangeTable, Item: item2}), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.should.eql({UnprocessedItems: {}}) + // request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.should.eql({}) + // request(helpers.opts('GetItem', {TableName: helpers.testRangeTable, Key: {a: item2.a, b: item2.b}, ConsistentRead: true}), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.should.eql({}) + // done() + // }) + // }) + // }) + // }) + // }) + // }) + // + // it('should deal with puts and deletes together', function(done) { + // var item = {a: {S: helpers.randomString()}, c: {S: 'c'}}, + // item2 = {a: {S: helpers.randomString()}, c: {S: 'c'}}, + // batchReq = {TransactItems: {}} + // request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {PutRequest: {Item: item2}}] + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.body.should.eql({UnprocessedItems: {}}) + // batchReq.TransactItems[helpers.testHashTable] = [{PutRequest: {Item: item}}, {DeleteRequest: {Key: {a: item2.a}}}] + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.body.should.eql({UnprocessedItems: {}}) + // request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.should.eql({Item: item}) + // request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.should.eql({}) + // done() + // }) + // }) + // }) + // }) + // }) + // }) + // + // it('should return ConsumedCapacity from each specified table when putting and deleting small item', function(done) { + // var a = helpers.randomString(), b = new Array(1010 - a.length).join('b'), + // item = {a: {S: a}, b: {S: b}, c: {N: '12.3456'}, d: {B: 'AQI='}, e: {BS: ['AQI=', 'Ag==', 'AQ==']}}, + // key2 = helpers.randomString(), key3 = helpers.randomNumber(), + // batchReq = {TransactItems: {}, ReturnConsumedCapacity: 'TOTAL'} + // batchReq.TransactItems[helpers.testHashTable] = [{PutRequest: {Item: item}}, {PutRequest: {Item: {a: {S: key2}}}}] + // batchReq.TransactItems[helpers.testHashNTable] = [{PutRequest: {Item: {a: {N: key3}}}}] + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) + // batchReq.ReturnConsumedCapacity = 'INDEXES' + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) + // batchReq.ReturnConsumedCapacity = 'TOTAL' + // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {DeleteRequest: {Key: {a: {S: key2}}}}] + // batchReq.TransactItems[helpers.testHashNTable] = [{DeleteRequest: {Key: {a: {N: key3}}}}] + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) + // batchReq.ReturnConsumedCapacity = 'INDEXES' + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) + // done() + // }) + // }) + // }) + // }) + // }) + // + // it('should return ConsumedCapacity from each specified table when putting and deleting larger item', function(done) { + // var a = helpers.randomString(), b = new Array(1012 - a.length).join('b'), + // item = {a: {S: a}, b: {S: b}, c: {N: '12.3456'}, d: {B: 'AQI='}, e: {BS: ['AQI=', 'Ag==']}}, + // key2 = helpers.randomString(), key3 = helpers.randomNumber(), + // batchReq = {TransactItems: {}, ReturnConsumedCapacity: 'TOTAL'} + // batchReq.TransactItems[helpers.testHashTable] = [{PutRequest: {Item: item}}, {PutRequest: {Item: {a: {S: key2}}}}] + // batchReq.TransactItems[helpers.testHashNTable] = [{PutRequest: {Item: {a: {N: key3}}}}] + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 3, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) + // batchReq.ReturnConsumedCapacity = 'INDEXES' + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 3, Table: {CapacityUnits: 3}, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) + // batchReq.ReturnConsumedCapacity = 'TOTAL' + // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {DeleteRequest: {Key: {a: {S: key2}}}}] + // batchReq.TransactItems[helpers.testHashNTable] = [{DeleteRequest: {Key: {a: {N: key3}}}}] + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 3, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) + // batchReq.ReturnConsumedCapacity = 'INDEXES' + // request(opts(batchReq), function(err, res) { + // if (err) return done(err) + // res.statusCode.should.equal(200) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashTable}) + // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) + // done() + // }) + // }) + // }) + // }) + // }) + // + // + // // All capacities seem to have a burst rate of 300x => full recovery is 300sec + // // Max size = 1638400 = 25 * 65536 = 1600 capacity units + // // Will process all if capacity >= 751. Below this value, the algorithm is something like: + // // min(capacity * 300, min(capacity, 336) + 677) + random(mean = 80, stddev = 32) + // it.skip('should return UnprocessedItems if over limit', function(done) { + // this.timeout(1e8) + // + // var CAPACITY = 3 + // + // async.times(10, createAndWrite, done) + // + // function createAndWrite(i, cb) { + // var name = helpers.randomName(), table = { + // TableName: name, + // AttributeDefinitions: [{AttributeName: 'a', AttributeType: 'S'}], + // KeySchema: [{KeyType: 'HASH', AttributeName: 'a'}], + // ProvisionedThroughput: {ReadCapacityUnits: CAPACITY, WriteCapacityUnits: CAPACITY}, + // } + // helpers.createAndWait(table, function(err) { + // if (err) return cb(err) + // async.timesSeries(50, function(n, cb) { batchWrite(name, n, cb) }, cb) + // }) + // } + // + // function batchWrite(name, n, cb) { + // var i, item, items = [], totalSize = 0, batchReq = {TransactItems: {}, ReturnConsumedCapacity: 'TOTAL'} + // + // for (i = 0; i < 25; i++) { + // item = {a: {S: ('0' + i).slice(-2)}, + // b: {S: new Array(Math.floor((64 - (16 * Math.random())) * 1024) - 3).join('b')}} + // totalSize += db.itemSize(item) + // items.push({PutRequest: {Item: item}}) + // } + // + // batchReq.TransactItems[name] = items + // request(opts(batchReq), function(err, res) { + // // if (err) return cb(err) + // if (err) { + // // console.log('Caught err: ' + err) + // return cb() + // } + // if (/ProvisionedThroughputExceededException$/.test(res.body.__type)) { + // // console.log('ProvisionedThroughputExceededException$') + // return cb() + // } else if (res.body.__type) { + // // return cb(new Error(JSON.stringify(res.body))) + // return cb() + // } + // res.statusCode.should.equal(200) + // // eslint-disable-next-line no-console + // console.log([CAPACITY, res.body.ConsumedCapacity[0].CapacityUnits, totalSize].join()) + // setTimeout(cb, res.body.ConsumedCapacity[0].CapacityUnits * 1000 / CAPACITY) + // }) + // } + }) + }) +}) \ No newline at end of file diff --git a/validations/transactWriteItems.js b/validations/transactWriteItems.js index 6126ab9..b66ccc3 100644 --- a/validations/transactWriteItems.js +++ b/validations/transactWriteItems.js @@ -17,6 +17,7 @@ exports.types = { type: 'List', notNull: true, lengthGreaterThanOrEqual: 1, + lengthLessThanOrEqual: 25, children: { type: 'ValueStruct', children: { @@ -74,6 +75,9 @@ exports.types = { notNull: true, children: 'AttrStruct', }, + UpdateExpression: { + type: 'String', + }, }, }, Delete: { @@ -136,3 +140,34 @@ exports.types = { }, }, } + +exports.custom = function(data, store) { + var i, request, msg + for (i = 0; i < data.TransactItems.length; i++) { + request = data.TransactItems[i] + if (request.Put) { + for (let key in request.Put.Item) { + msg = validations.validateAttributeValue(request.Put.Item[key]) + if (msg) return msg + } + if (db.itemSize(request.Put.Item) > store.options.maxItemSize) + return 'Item size has exceeded the maximum allowed size' + } else if (request.Delete) { + msg = validations.validateAttributeValue(request.Delete.Key) + if (msg) return msg + } else if (request.Update) { + var msg = validations.validateExpressionParams(request.Update, + ['UpdateExpression', 'ConditionExpression'], + ['AttributeUpdates', 'Expected']) + if (msg) return msg + msg = validations.validateAttributeValue(request.Update.Key) + if (msg) return msg + msg = validations.validateAttributeConditions(request.Update) + if (msg) return msg + msg = validations.validateExpressions(request.Update) + if (msg) return msg + } else { + return 'The action or operation requested is invalid. Verify that the action is typed correctly.' + } + } +} From ff3faa57c3f9d5d9183f8e90d0ca9e3e4468fa77 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Wed, 28 Jul 2021 14:00:27 -0400 Subject: [PATCH 03/12] implement atomic transactions --- actions/transactWriteItems.js | 191 ++++++++++++++++++++++++---------- actions/updateItem.js | 8 +- test/transactWriteItem.js | 2 +- 3 files changed, 146 insertions(+), 55 deletions(-) diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index b0ed9e5..44d92ef 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -1,110 +1,195 @@ var async = require('async'), - putItem = require('./putItem'), - deleteItem = require('./deleteItem'), - updateItem = require('./updateItem'), + { applyAttributeUpdates, applyUpdateExpression, deepClone } = require('./updateItem'), db = require('../db') module.exports = function transactWriteItem(store, data, cb) { - var actions = [] var seenKeys = {} + var batchOpts = {} + var releaseLocks = [] - async.series([ - async.eachSeries.bind(async, data.TransactItems, addActions), - async.series.bind(async, actions), - ], function (err, responses) { - console.log('wake up at 11:30') - console.dir(err) - console.dir(responses) - + async.eachSeries(data.TransactItems, addActions, function (err, responses) { if (err) { if (err.body && (/Missing the key/.test(err.body.message) || /Type mismatch for key/.test(err.body.message))) err.body.message = 'The provided key element does not match the schema' return cb(err) } - var res = {UnprocessedItems: {}}, tableUnits = {} - if (~['TOTAL', 'INDEXES'].indexOf(data.ReturnConsumedCapacity)) { - responses[1].forEach(function (action) { - var table = action.ConsumedCapacity.TableName - if (!tableUnits[table]) tableUnits[table] = 0 - tableUnits[table] += action.ConsumedCapacity.CapacityUnits - }) - res.ConsumedCapacity = Object.keys(tableUnits).map(function (table) { - return { - CapacityUnits: tableUnits[table], - TableName: table, - Table: data.ReturnConsumedCapacity == 'INDEXES' ? {CapacityUnits: tableUnits[table]} : undefined, - } - }) - } + // this does NOT ensure atomicity across tables - but the items on each table are already locked + // and the actions have been validated. at this point the only thing that would fail would be the + // database itself, and that's a lot of work to get around so I'm just ¯\_(ツ)_/¯ not gonna do that + var operationsbyTable = Object.entries(batchOpts) - cb(null, res) + async.each(operationsbyTable, function([tableName, ops], callback) { + var itemDb = store.getItemDb(tableName) + itemDb.batch(ops, function(err, results) { + if (err) callback(err) + callback(results) + }) + }, function(err, results) { + async.parallel(releaseLocks) + if (err) return cb(err) + + var res = {UnprocessedItems: {}}, tableUnits = {} + + if (~['TOTAL', 'INDEXES'].indexOf(data.ReturnConsumedCapacity)) { + responses[1].forEach(function (action) { + var table = action.ConsumedCapacity.TableName + if (!tableUnits[table]) tableUnits[table] = 0 + tableUnits[table] += action.ConsumedCapacity.CapacityUnits + }) + res.ConsumedCapacity = Object.keys(tableUnits).map(function (table) { + return { + CapacityUnits: tableUnits[table], + TableName: table, + Table: data.ReturnConsumedCapacity == 'INDEXES' ? {CapacityUnits: tableUnits[table]} : undefined, + } + }) + } + + cb(null, res) + }) }) function addActions(transactItem, cb) { var options = {} var tableName - var key if (data.ReturnConsumedCapacity) options.ReturnConsumedCapacity = data.ReturnConsumedCapacity if (transactItem.Put) { - tableName = transactItem.Put.TableName; - options = {TableName: tableName} + tableName = transactItem.Put.TableName store.getTable(tableName, function (err, table) { if (err) return cb(err) if ((err = db.validateItem(transactItem.Put.Item, table)) != null) return cb(err) - options.Item = transactItem.Put.Item - actions.push(putItem.bind(null, store, options)) - key = db.createKey(options.Item, table) - + let value = transactItem.Put.Item + let key = db.createKey(transactItem.Put.Item, table) if (seenKeys[key]) { return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) } + seenKeys[key] = true - return cb() + + var itemDb = store.getItemDb(tableName) + + itemDb.get(key, function(err, oldItem) { + if (oldItem) { + itemDb.lock(key, function(release) { + releaseLocks.push(release) + console.dir(release) + }) + } + + if (err && err.name != 'NotFoundError') return cb(err) + + if ((err = db.checkConditional(transactItem.Put, oldItem)) != null) return cb(err) + + let operation = { + type: 'put', + key, + value + } + + if (batchOpts[tableName]) { + batchOpts[tableName].push(operation) + } else { + batchOpts[tableName] = [operation] + } + + return cb() + }) }) } else if (transactItem.Delete) { - tableName = transactItem.Delete.TableName; - options = {TableName: tableName} + tableName = transactItem.Delete.TableName store.getTable(tableName, function (err, table) { if (err) return cb(err) if ((err = db.validateKey(transactItem.Delete.Key, table) != null)) return cb(err) - options.Key = transactItem.Delete.Key - actions.push(deleteItem.bind(null, store, options)) - - key = db.createKey(options.Key, table) - + let key = db.createKey(transactItem.Delete.Key, table) if (seenKeys[key]) { return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) } + seenKeys[key] = true - return cb() + + var itemDb = store.getItemDb(tableName) + + itemDb.lock(key, function(release) { + releaseLocks.push(release) + itemDb.get(key, function(err, oldItem) { + if (err && err.name != 'NotFoundError') return cb(err) + + if ((err = db.checkConditional(transactItem.Delete, oldItem)) != null) return cb(err) + + let operation = { + type: 'del', + key + } + + if (batchOpts[tableName]) { + batchOpts[tableName].push(operation) + } else { + batchOpts[tableName] = [operation] + } + return cb() + }) + }) }) } else if (transactItem.Update) { - tableName = transactItem.Update.TableName; - options = transactItem.Update + tableName = transactItem.Update.TableName store.getTable(tableName, function (err, table) { if (err) return cb(err) - if ((err = db.validateKey(transactItem.Update.Key, table) != null)) return cb(err) - options.Key = transactItem.Update.Key - actions.push(updateItem.bind(null, store, options)) - - key = db.createKey(options.Key, table) + if ((err = db.validateUpdates(transactItem.Update.AttributeUpdates, transactItem.Update._updates, table)) != null) return cb(err) + let key = db.createKey(transactItem.Update.Key, table) if (seenKeys[key]) { return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) } + seenKeys[key] = true - return cb() + + var itemDb = store.getItemDb(tableName) + + itemDb.lock(key, function(release) { + releaseLocks.push(release) + itemDb.get(key, function(err, oldItem) { + if (err && err.name != 'NotFoundError') return cb(err) + + if ((err = db.checkConditional(transactItem.Update, oldItem)) != null) return cb(err) + + var item = transactItem.Update.Key + + if (oldItem) { + item = deepClone(oldItem) + } + + err = transactItem.Update._updates ? applyUpdateExpression(transactItem.Update._updates.sections, table, item) : + applyAttributeUpdates(transactItem.Update.AttributeUpdates, table, item) + if (err) return cb(err) + + if (db.itemSize(item) > store.options.maxItemSize) + return cb(db.validationError('Item size to update has exceeded the maximum allowed size')) + + let operation = { + type: 'put', + key, + value: item + } + + if (batchOpts[tableName]) { + batchOpts[tableName].push(operation) + } else { + batchOpts[tableName] = [operation] + } + + return cb() + }) + }) }) } - } } diff --git a/actions/updateItem.js b/actions/updateItem.js index 72f7f7c..55c5f27 100644 --- a/actions/updateItem.js +++ b/actions/updateItem.js @@ -1,7 +1,13 @@ var Big = require('big.js'), db = require('../db') -module.exports = function updateItem(store, data, cb) { +exports.deepClone = deepClone +exports.applyAttributeUpdates = applyAttributeUpdates +exports.applyUpdateExpression = applyUpdateExpression +exports.resolveValue = resolveValue +exports.deleteFromParent = deleteFromParent +exports.addToParent = addToParent +exports.updateItem = function updateItem(store, data, cb) { store.getTable(data.TableName, function(err, table) { if (err) return cb(err) diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js index dfeb11a..5fc2241 100644 --- a/test/transactWriteItem.js +++ b/test/transactWriteItem.js @@ -479,7 +479,7 @@ describe('transactWriteItem', function() { // update item if (err) return done(err) res.statusCode.should.equal(200) - res.body.Item.should.equal({}) + res.body.should.eql({}) request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { // put item if (err) return done(err) From c2d001f073fa6f230032d91721727934ad5bcea8 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Wed, 28 Jul 2021 14:34:04 -0400 Subject: [PATCH 04/12] implement ConsumedCapacity --- actions/transactWriteItems.js | 16 +- test/transactWriteItem.js | 304 ++++++++++++---------------------- 2 files changed, 116 insertions(+), 204 deletions(-) diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index 44d92ef..15b15f5 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -7,7 +7,7 @@ module.exports = function transactWriteItem(store, data, cb) { var batchOpts = {} var releaseLocks = [] - async.eachSeries(data.TransactItems, addActions, function (err, responses) { + async.eachSeries(data.TransactItems, addActions, function (err) { if (err) { if (err.body && (/Missing the key/.test(err.body.message) || /Type mismatch for key/.test(err.body.message))) err.body.message = 'The provided key element does not match the schema' @@ -25,17 +25,20 @@ module.exports = function transactWriteItem(store, data, cb) { if (err) callback(err) callback(results) }) - }, function(err, results) { + }, function(err) { async.parallel(releaseLocks) if (err) return cb(err) var res = {UnprocessedItems: {}}, tableUnits = {} if (~['TOTAL', 'INDEXES'].indexOf(data.ReturnConsumedCapacity)) { - responses[1].forEach(function (action) { - var table = action.ConsumedCapacity.TableName - if (!tableUnits[table]) tableUnits[table] = 0 - tableUnits[table] += action.ConsumedCapacity.CapacityUnits + operationsbyTable.forEach(([table, operations]) => { + tableUnits[table] = 0 + operations.forEach(op => { + let readCapacity = db.capacityUnits(op.value, true, true) + let writeCapacity = db.capacityUnits(op.value, false, true) + tableUnits[table] += readCapacity + writeCapacity + }) }) res.ConsumedCapacity = Object.keys(tableUnits).map(function (table) { return { @@ -77,7 +80,6 @@ module.exports = function transactWriteItem(store, data, cb) { if (oldItem) { itemDb.lock(key, function(release) { releaseLocks.push(release) - console.dir(release) }) } diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js index 5fc2241..9b98010 100644 --- a/test/transactWriteItem.js +++ b/test/transactWriteItem.js @@ -491,203 +491,113 @@ describe('transactWriteItem', function() { }) }) }) - // - // it('should delete an item from each table', function(done) { - // var item = {a: {S: helpers.randomString()}, c: {S: 'c'}}, - // item2 = {a: {S: helpers.randomString()}, b: {S: helpers.randomString()}, c: {S: 'c'}}, - // batchReq = {TransactItems: {}} - // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}] - // batchReq.TransactItems[helpers.testRangeTable] = [{DeleteRequest: {Key: {a: item2.a, b: item2.b}}}] - // request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // request(helpers.opts('PutItem', {TableName: helpers.testRangeTable, Item: item2}), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.should.eql({UnprocessedItems: {}}) - // request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.should.eql({}) - // request(helpers.opts('GetItem', {TableName: helpers.testRangeTable, Key: {a: item2.a, b: item2.b}, ConsistentRead: true}), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.should.eql({}) - // done() - // }) - // }) - // }) - // }) - // }) - // }) - // - // it('should deal with puts and deletes together', function(done) { - // var item = {a: {S: helpers.randomString()}, c: {S: 'c'}}, - // item2 = {a: {S: helpers.randomString()}, c: {S: 'c'}}, - // batchReq = {TransactItems: {}} - // request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {PutRequest: {Item: item2}}] - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.body.should.eql({UnprocessedItems: {}}) - // batchReq.TransactItems[helpers.testHashTable] = [{PutRequest: {Item: item}}, {DeleteRequest: {Key: {a: item2.a}}}] - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.body.should.eql({UnprocessedItems: {}}) - // request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.should.eql({Item: item}) - // request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.should.eql({}) - // done() - // }) - // }) - // }) - // }) - // }) - // }) - // - // it('should return ConsumedCapacity from each specified table when putting and deleting small item', function(done) { - // var a = helpers.randomString(), b = new Array(1010 - a.length).join('b'), - // item = {a: {S: a}, b: {S: b}, c: {N: '12.3456'}, d: {B: 'AQI='}, e: {BS: ['AQI=', 'Ag==', 'AQ==']}}, - // key2 = helpers.randomString(), key3 = helpers.randomNumber(), - // batchReq = {TransactItems: {}, ReturnConsumedCapacity: 'TOTAL'} - // batchReq.TransactItems[helpers.testHashTable] = [{PutRequest: {Item: item}}, {PutRequest: {Item: {a: {S: key2}}}}] - // batchReq.TransactItems[helpers.testHashNTable] = [{PutRequest: {Item: {a: {N: key3}}}}] - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) - // batchReq.ReturnConsumedCapacity = 'INDEXES' - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) - // batchReq.ReturnConsumedCapacity = 'TOTAL' - // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {DeleteRequest: {Key: {a: {S: key2}}}}] - // batchReq.TransactItems[helpers.testHashNTable] = [{DeleteRequest: {Key: {a: {N: key3}}}}] - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) - // batchReq.ReturnConsumedCapacity = 'INDEXES' - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) - // done() - // }) - // }) - // }) - // }) - // }) - // - // it('should return ConsumedCapacity from each specified table when putting and deleting larger item', function(done) { - // var a = helpers.randomString(), b = new Array(1012 - a.length).join('b'), - // item = {a: {S: a}, b: {S: b}, c: {N: '12.3456'}, d: {B: 'AQI='}, e: {BS: ['AQI=', 'Ag==']}}, - // key2 = helpers.randomString(), key3 = helpers.randomNumber(), - // batchReq = {TransactItems: {}, ReturnConsumedCapacity: 'TOTAL'} - // batchReq.TransactItems[helpers.testHashTable] = [{PutRequest: {Item: item}}, {PutRequest: {Item: {a: {S: key2}}}}] - // batchReq.TransactItems[helpers.testHashNTable] = [{PutRequest: {Item: {a: {N: key3}}}}] - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 3, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) - // batchReq.ReturnConsumedCapacity = 'INDEXES' - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 3, Table: {CapacityUnits: 3}, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) - // batchReq.ReturnConsumedCapacity = 'TOTAL' - // batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {DeleteRequest: {Key: {a: {S: key2}}}}] - // batchReq.TransactItems[helpers.testHashNTable] = [{DeleteRequest: {Key: {a: {N: key3}}}}] - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 3, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, TableName: helpers.testHashNTable}) - // batchReq.ReturnConsumedCapacity = 'INDEXES' - // request(opts(batchReq), function(err, res) { - // if (err) return done(err) - // res.statusCode.should.equal(200) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashTable}) - // res.body.ConsumedCapacity.should.containEql({CapacityUnits: 1, Table: {CapacityUnits: 1}, TableName: helpers.testHashNTable}) - // done() - // }) - // }) - // }) - // }) - // }) - // - // - // // All capacities seem to have a burst rate of 300x => full recovery is 300sec - // // Max size = 1638400 = 25 * 65536 = 1600 capacity units - // // Will process all if capacity >= 751. Below this value, the algorithm is something like: - // // min(capacity * 300, min(capacity, 336) + 677) + random(mean = 80, stddev = 32) - // it.skip('should return UnprocessedItems if over limit', function(done) { - // this.timeout(1e8) - // - // var CAPACITY = 3 - // - // async.times(10, createAndWrite, done) - // - // function createAndWrite(i, cb) { - // var name = helpers.randomName(), table = { - // TableName: name, - // AttributeDefinitions: [{AttributeName: 'a', AttributeType: 'S'}], - // KeySchema: [{KeyType: 'HASH', AttributeName: 'a'}], - // ProvisionedThroughput: {ReadCapacityUnits: CAPACITY, WriteCapacityUnits: CAPACITY}, - // } - // helpers.createAndWait(table, function(err) { - // if (err) return cb(err) - // async.timesSeries(50, function(n, cb) { batchWrite(name, n, cb) }, cb) - // }) - // } - // - // function batchWrite(name, n, cb) { - // var i, item, items = [], totalSize = 0, batchReq = {TransactItems: {}, ReturnConsumedCapacity: 'TOTAL'} - // - // for (i = 0; i < 25; i++) { - // item = {a: {S: ('0' + i).slice(-2)}, - // b: {S: new Array(Math.floor((64 - (16 * Math.random())) * 1024) - 3).join('b')}} - // totalSize += db.itemSize(item) - // items.push({PutRequest: {Item: item}}) - // } - // - // batchReq.TransactItems[name] = items - // request(opts(batchReq), function(err, res) { - // // if (err) return cb(err) - // if (err) { - // // console.log('Caught err: ' + err) - // return cb() - // } - // if (/ProvisionedThroughputExceededException$/.test(res.body.__type)) { - // // console.log('ProvisionedThroughputExceededException$') - // return cb() - // } else if (res.body.__type) { - // // return cb(new Error(JSON.stringify(res.body))) - // return cb() - // } - // res.statusCode.should.equal(200) - // // eslint-disable-next-line no-console - // console.log([CAPACITY, res.body.ConsumedCapacity[0].CapacityUnits, totalSize].join()) - // setTimeout(cb, res.body.ConsumedCapacity[0].CapacityUnits * 1000 / CAPACITY) - // }) - // } + + it('should write to two different tables', function(done) { + var hashItem = {a: {S: helpers.randomString()}, c: {S: 'c'}}, + rangeItem = {a: {S: helpers.randomString()}, b: {S: helpers.randomString()}, g: {N: '23'}} + transactReq = {TransactItems: []} + transactReq.TransactItems = [{Put: {TableName: helpers.testHashTable, Item: hashItem}}, {Put: {TableName: helpers.testRangeTable, Item: rangeItem}}] + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: hashItem.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({Item: hashItem}) + request(helpers.opts('GetItem', {TableName: helpers.testRangeTable, Key: {a: rangeItem.a, b: rangeItem.b}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({Item: rangeItem}) + done() + }) + }) + }) + }) + + it('should return ConsumedCapacity from each specified table when putting and deleting small item', function(done) { + var a = helpers.randomString(), b = new Array(1010 - a.length).join('b'), + item = {a: {S: a}, b: {S: b}, c: {N: '12.3456'}, d: {B: 'AQI='}, e: {BS: ['AQI=', 'Ag==', 'AQ==']}}, + key2 = helpers.randomString(), key3 = helpers.randomNumber(), + batchReq = {TransactItems: {}, ReturnConsumedCapacity: 'TOTAL'} + batchReq.TransactItems = [ + {Put: {Item: item, TableName: helpers.testHashTable}}, + {Put: {Item: {a: {S: key2}}, TableName: helpers.testHashTable}}, + {Put: {Item: {a: {N: key3}}, TableName: helpers.testHashNTable}} + ] + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 4, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashNTable}) + batchReq.ReturnConsumedCapacity = 'INDEXES' + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 4, Table: {CapacityUnits: 4}, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashNTable}) + batchReq.ReturnConsumedCapacity = 'TOTAL' + batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {DeleteRequest: {Key: {a: {S: key2}}}}] + batchReq.TransactItems[helpers.testHashNTable] = [{DeleteRequest: {Key: {a: {N: key3}}}}] + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 4, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashNTable}) + batchReq.ReturnConsumedCapacity = 'INDEXES' + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 4, Table: {CapacityUnits: 4}, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashNTable}) + done() + }) + }) + }) + }) + }) + + it('should return ConsumedCapacity from each specified table when putting and deleting larger item', function(done) { + var a = helpers.randomString(), b = new Array(1012 - a.length).join('b'), + item = {a: {S: a}, b: {S: b}, c: {N: '12.3456'}, d: {B: 'AQI='}, e: {BS: ['AQI=', 'Ag==']}}, + key2 = helpers.randomString(), key3 = helpers.randomNumber(), + batchReq = {TransactItems: [], ReturnConsumedCapacity: 'TOTAL'} + batchReq.TransactItems = [ + {Put: {Item: item, TableName: helpers.testHashTable}}, + {Put: {Item: {a: {S: key2}}, TableName: helpers.testHashTable}}, + {Put: {Item: {a: {N: key3}}, TableName: helpers.testHashNTable}} + ] + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 5, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashNTable}) + batchReq.ReturnConsumedCapacity = 'INDEXES' + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 5, Table: {CapacityUnits: 5}, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashNTable}) + batchReq.ReturnConsumedCapacity = 'TOTAL' + batchReq.TransactItems[helpers.testHashTable] = [{DeleteRequest: {Key: {a: item.a}}}, {DeleteRequest: {Key: {a: {S: key2}}}}] + batchReq.TransactItems[helpers.testHashNTable] = [{DeleteRequest: {Key: {a: {N: key3}}}}] + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 5, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, TableName: helpers.testHashNTable}) + batchReq.ReturnConsumedCapacity = 'INDEXES' + request(opts(batchReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 5, Table: {CapacityUnits: 5}, TableName: helpers.testHashTable}) + res.body.ConsumedCapacity.should.containEql({CapacityUnits: 2, Table: {CapacityUnits: 2}, TableName: helpers.testHashNTable}) + done() + }) + }) + }) + }) + }) }) }) }) \ No newline at end of file From 151409748a939d059ad6ecac8386d6866eadc6d8 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Thu, 29 Jul 2021 09:29:34 -0400 Subject: [PATCH 05/12] fix updateItem exports --- actions/transactWriteItems.js | 6 +- actions/updateItem.js | 193 +--------------------------------- db/index.js | 182 ++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 191 deletions(-) diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index 15b15f5..0680079 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -166,11 +166,11 @@ module.exports = function transactWriteItem(store, data, cb) { var item = transactItem.Update.Key if (oldItem) { - item = deepClone(oldItem) + item = db.deepClone(oldItem) } - err = transactItem.Update._updates ? applyUpdateExpression(transactItem.Update._updates.sections, table, item) : - applyAttributeUpdates(transactItem.Update.AttributeUpdates, table, item) + err = transactItem.Update._updates ? db.applyUpdateExpression(transactItem.Update._updates.sections, table, item) : + db.applyAttributeUpdates(transactItem.Update.AttributeUpdates, table, item) if (err) return cb(err) if (db.itemSize(item) > store.options.maxItemSize) diff --git a/actions/updateItem.js b/actions/updateItem.js index 55c5f27..1e91501 100644 --- a/actions/updateItem.js +++ b/actions/updateItem.js @@ -1,13 +1,6 @@ -var Big = require('big.js'), - db = require('../db') +var db = require('../db') -exports.deepClone = deepClone -exports.applyAttributeUpdates = applyAttributeUpdates -exports.applyUpdateExpression = applyUpdateExpression -exports.resolveValue = resolveValue -exports.deleteFromParent = deleteFromParent -exports.addToParent = addToParent -exports.updateItem = function updateItem(store, data, cb) { +module.exports = function updateItem(store, data, cb) { store.getTable(data.TableName, function(err, table) { if (err) return cb(err) @@ -30,7 +23,7 @@ exports.updateItem = function updateItem(store, data, cb) { paths = data._updates ? data._updates.paths : Object.keys(data.AttributeUpdates || {}) if (oldItem) { - item = deepClone(oldItem) + item = db.deepClone(oldItem) if (data.ReturnValues == 'ALL_OLD') { returnObj.Attributes = oldItem } else if (data.ReturnValues == 'UPDATED_OLD') { @@ -38,8 +31,8 @@ exports.updateItem = function updateItem(store, data, cb) { } } - err = data._updates ? applyUpdateExpression(data._updates.sections, table, item) : - applyAttributeUpdates(data.AttributeUpdates, table, item) + err = data._updates ? db.applyUpdateExpression(data._updates.sections, table, item) : + db.applyAttributeUpdates(data.AttributeUpdates, table, item) if (err) return cb(err) if (db.itemSize(item) > store.options.maxItemSize) @@ -65,179 +58,3 @@ exports.updateItem = function updateItem(store, data, cb) { }) }) } - -// Relatively fast deep clone of simple objects/arrays -function deepClone(obj) { - if (typeof obj != 'object' || obj == null) return obj - var result - if (Array.isArray(obj)) { - result = new Array(obj.length) - for (var i = 0; i < obj.length; i++) { - result[i] = deepClone(obj[i]) - } - } else { - result = Object.create(null) - for (var attr in obj) { - result[attr] = deepClone(obj[attr]) - } - } - return result -} - -function applyAttributeUpdates(updates, table, item) { - for (var attr in updates) { - var update = updates[attr] - if (update.Action == 'PUT' || update.Action == null) { - item[attr] = update.Value - } else if (update.Action == 'ADD') { - if (update.Value.N) { - if (item[attr] && !item[attr].N) - return db.validationError('Type mismatch for attribute to update') - if (!item[attr]) item[attr] = {N: '0'} - item[attr].N = new Big(item[attr].N).plus(update.Value.N).toFixed() - } else { - var type = Object.keys(update.Value)[0] - if (item[attr] && !item[attr][type]) - return db.validationError('Type mismatch for attribute to update') - if (!item[attr]) item[attr] = {} - if (!item[attr][type]) item[attr][type] = [] - var val = type == 'L' ? update.Value[type] : update.Value[type].filter(function(a) { // eslint-disable-line no-loop-func - return !~item[attr][type].indexOf(a) - }) - item[attr][type] = item[attr][type].concat(val) - } - } else if (update.Action == 'DELETE') { - if (update.Value) { - type = Object.keys(update.Value)[0] - if (item[attr] && !item[attr][type]) - return db.validationError('Type mismatch for attribute to update') - if (item[attr] && item[attr][type]) { - item[attr][type] = item[attr][type].filter(function(val) { // eslint-disable-line no-loop-func - return !~update.Value[type].indexOf(val) - }) - if (!item[attr][type].length) delete item[attr] - } - } else { - delete item[attr] - } - } - } -} - -function applyUpdateExpression(sections, table, item) { - var toSquash = [] - for (var i = 0; i < sections.length; i++) { - var section = sections[i] - if (section.type == 'set') { - section.val = resolveValue(section.val, item) - if (typeof section.val == 'string') { - return db.validationError(section.val) - } - } - } - for (i = 0; i < sections.length; i++) { - section = sections[i] - var parent = db.mapPath(section.path.slice(0, -1), item) - var attr = section.path[section.path.length - 1] - if (parent == null || (typeof attr == 'number' ? parent.L : parent.M) == null) { - return db.validationError('The document path provided in the update expression is invalid for update') - } - var existing = parent.M ? parent.M[attr] : parent.L[attr] - var alreadyExists = existing != null - if (section.type == 'remove') { - deleteFromParent(parent, attr) - } else if (section.type == 'delete') { - if (alreadyExists && Object.keys(existing)[0] != section.attrType) { - return db.validationError('An operand in the update expression has an incorrect data type') - } - if (alreadyExists) { - existing[section.attrType] = existing[section.attrType].filter(function(val) { // eslint-disable-line no-loop-func - return !~section.val[section.attrType].indexOf(val) - }) - if (!existing[section.attrType].length) { - deleteFromParent(parent, attr) - } - } - } else if (section.type == 'add') { - if (alreadyExists && Object.keys(existing)[0] != section.attrType) { - return db.validationError('An operand in the update expression has an incorrect data type') - } - if (section.attrType == 'N') { - if (!existing) existing = {N: '0'} - existing.N = new Big(existing.N).plus(section.val.N).toFixed() - } else { - if (!existing) existing = {} - if (!existing[section.attrType]) existing[section.attrType] = [] - existing[section.attrType] = existing[section.attrType].concat(section.val[section.attrType].filter(function(a) { // eslint-disable-line no-loop-func - return !~existing[section.attrType].indexOf(a) - })) - } - if (!alreadyExists) { - addToParent(parent, attr, existing, toSquash) - } - } else if (section.type == 'set') { - if (section.path.length == 1) { - var err = db.traverseIndexes(table, function(attr, type) { - if (section.path[0] == attr && section.val[type] == null) { - return db.validationError('The update expression attempted to update the secondary index key to unsupported type') - } - }) - if (err) return err - } - addToParent(parent, attr, section.val, toSquash) - } - } - toSquash.forEach(function(obj) { obj.L = obj.L.filter(Boolean) }) -} - -function resolveValue(val, item) { - if (Array.isArray(val)) { - val = db.mapPath(val, item) - } else if (val.type == 'add' || val.type == 'subtract') { - var val1 = resolveValue(val.args[0], item) - if (typeof val1 == 'string') return val1 - if (val1.N == null) { - return 'An operand in the update expression has an incorrect data type' - } - var val2 = resolveValue(val.args[1], item) - if (typeof val2 == 'string') return val2 - if (val2.N == null) { - return 'An operand in the update expression has an incorrect data type' - } - val = {N: new Big(val1.N)[val.type == 'add' ? 'plus' : 'minus'](val2.N).toFixed()} - } else if (val.type == 'function' && val.name == 'if_not_exists') { - val = db.mapPath(val.args[0], item) || resolveValue(val.args[1], item) - } else if (val.type == 'function' && val.name == 'list_append') { - val1 = resolveValue(val.args[0], item) - if (typeof val1 == 'string') return val1 - if (val1.L == null) { - return 'An operand in the update expression has an incorrect data type' - } - val2 = resolveValue(val.args[1], item) - if (typeof val2 == 'string') return val2 - if (val2.L == null) { - return 'An operand in the update expression has an incorrect data type' - } - return {L: val1.L.concat(val2.L)} - } - return val || 'The provided expression refers to an attribute that does not exist in the item' -} - -function deleteFromParent(parent, attr) { - if (parent.M) { - delete parent.M[attr] - } else if (parent.L) { - parent.L.splice(attr, 1) - } -} - -function addToParent(parent, attr, val, toSquash) { - if (parent.M) { - parent.M[attr] = val - } else if (parent.L) { - if (attr > parent.L.length && !~toSquash.indexOf(parent)) { - toSquash.push(parent) - } - parent.L[attr] = val - } -} diff --git a/db/index.js b/db/index.js index 8bc13a8..a5cd57c 100644 --- a/db/index.js +++ b/db/index.js @@ -39,6 +39,12 @@ exports.mapPath = mapPath exports.queryTable = queryTable exports.updateIndexes = updateIndexes exports.getIndexActions = getIndexActions +exports.deepClone = deepClone +exports.applyAttributeUpdates = applyAttributeUpdates +exports.applyUpdateExpression = applyUpdateExpression +exports.resolveValue = resolveValue +exports.deleteFromParent = deleteFromParent +exports.addToParent = addToParent function create(options) { options = options || {} @@ -982,3 +988,179 @@ function getIndexActions(indexes, existingItem, item, table) { }) return {puts: puts, deletes: deletes} } + +// Relatively fast deep clone of simple objects/arrays +function deepClone(obj) { + if (typeof obj != 'object' || obj == null) return obj + var result + if (Array.isArray(obj)) { + result = new Array(obj.length) + for (var i = 0; i < obj.length; i++) { + result[i] = deepClone(obj[i]) + } + } else { + result = Object.create(null) + for (var attr in obj) { + result[attr] = deepClone(obj[attr]) + } + } + return result +} + +function applyAttributeUpdates(updates, table, item) { + for (var attr in updates) { + var update = updates[attr] + if (update.Action == 'PUT' || update.Action == null) { + item[attr] = update.Value + } else if (update.Action == 'ADD') { + if (update.Value.N) { + if (item[attr] && !item[attr].N) + return validationError('Type mismatch for attribute to update') + if (!item[attr]) item[attr] = {N: '0'} + item[attr].N = new Big(item[attr].N).plus(update.Value.N).toFixed() + } else { + var type = Object.keys(update.Value)[0] + if (item[attr] && !item[attr][type]) + return validationError('Type mismatch for attribute to update') + if (!item[attr]) item[attr] = {} + if (!item[attr][type]) item[attr][type] = [] + var val = type == 'L' ? update.Value[type] : update.Value[type].filter(function(a) { // eslint-disable-line no-loop-func + return !~item[attr][type].indexOf(a) + }) + item[attr][type] = item[attr][type].concat(val) + } + } else if (update.Action == 'DELETE') { + if (update.Value) { + type = Object.keys(update.Value)[0] + if (item[attr] && !item[attr][type]) + return validationError('Type mismatch for attribute to update') + if (item[attr] && item[attr][type]) { + item[attr][type] = item[attr][type].filter(function(val) { // eslint-disable-line no-loop-func + return !~update.Value[type].indexOf(val) + }) + if (!item[attr][type].length) delete item[attr] + } + } else { + delete item[attr] + } + } + } +} + +function applyUpdateExpression(sections, table, item) { + var toSquash = [] + for (var i = 0; i < sections.length; i++) { + var section = sections[i] + if (section.type == 'set') { + section.val = resolveValue(section.val, item) + if (typeof section.val == 'string') { + return validationError(section.val) + } + } + } + for (i = 0; i < sections.length; i++) { + section = sections[i] + var parent = mapPath(section.path.slice(0, -1), item) + var attr = section.path[section.path.length - 1] + if (parent == null || (typeof attr == 'number' ? parent.L : parent.M) == null) { + return validationError('The document path provided in the update expression is invalid for update') + } + var existing = parent.M ? parent.M[attr] : parent.L[attr] + var alreadyExists = existing != null + if (section.type == 'remove') { + deleteFromParent(parent, attr) + } else if (section.type == 'delete') { + if (alreadyExists && Object.keys(existing)[0] != section.attrType) { + return validationError('An operand in the update expression has an incorrect data type') + } + if (alreadyExists) { + existing[section.attrType] = existing[section.attrType].filter(function(val) { // eslint-disable-line no-loop-func + return !~section.val[section.attrType].indexOf(val) + }) + if (!existing[section.attrType].length) { + deleteFromParent(parent, attr) + } + } + } else if (section.type == 'add') { + if (alreadyExists && Object.keys(existing)[0] != section.attrType) { + return validationError('An operand in the update expression has an incorrect data type') + } + if (section.attrType == 'N') { + if (!existing) existing = {N: '0'} + existing.N = new Big(existing.N).plus(section.val.N).toFixed() + } else { + if (!existing) existing = {} + if (!existing[section.attrType]) existing[section.attrType] = [] + existing[section.attrType] = existing[section.attrType].concat(section.val[section.attrType].filter(function(a) { // eslint-disable-line no-loop-func + return !~existing[section.attrType].indexOf(a) + })) + } + if (!alreadyExists) { + addToParent(parent, attr, existing, toSquash) + } + } else if (section.type == 'set') { + if (section.path.length == 1) { + var err = traverseIndexes(table, function(attr, type) { + if (section.path[0] == attr && section.val[type] == null) { + return validationError('The update expression attempted to update the secondary index key to unsupported type') + } + }) + if (err) return err + } + addToParent(parent, attr, section.val, toSquash) + } + } + toSquash.forEach(function(obj) { obj.L = obj.L.filter(Boolean) }) +} + +function resolveValue(val, item) { + if (Array.isArray(val)) { + val = mapPath(val, item) + } else if (val.type == 'add' || val.type == 'subtract') { + var val1 = resolveValue(val.args[0], item) + if (typeof val1 == 'string') return val1 + if (val1.N == null) { + return 'An operand in the update expression has an incorrect data type' + } + var val2 = resolveValue(val.args[1], item) + if (typeof val2 == 'string') return val2 + if (val2.N == null) { + return 'An operand in the update expression has an incorrect data type' + } + val = {N: new Big(val1.N)[val.type == 'add' ? 'plus' : 'minus'](val2.N).toFixed()} + } else if (val.type == 'function' && val.name == 'if_not_exists') { + val = mapPath(val.args[0], item) || resolveValue(val.args[1], item) + } else if (val.type == 'function' && val.name == 'list_append') { + val1 = resolveValue(val.args[0], item) + if (typeof val1 == 'string') return val1 + if (val1.L == null) { + return 'An operand in the update expression has an incorrect data type' + } + val2 = resolveValue(val.args[1], item) + if (typeof val2 == 'string') return val2 + if (val2.L == null) { + return 'An operand in the update expression has an incorrect data type' + } + return {L: val1.L.concat(val2.L)} + } + return val || 'The provided expression refers to an attribute that does not exist in the item' +} + +function deleteFromParent(parent, attr) { + if (parent.M) { + delete parent.M[attr] + } else if (parent.L) { + parent.L.splice(attr, 1) + } +} + +function addToParent(parent, attr, val, toSquash) { + if (parent.M) { + parent.M[attr] = val + } else if (parent.L) { + if (attr > parent.L.length && !~toSquash.indexOf(parent)) { + toSquash.push(parent) + } + parent.L[attr] = val + } +} \ No newline at end of file From e00f8f4c2d38a63e1b360a0db303a6c37d54e951 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Thu, 29 Jul 2021 10:16:17 -0400 Subject: [PATCH 06/12] tests++ --- actions/transactWriteItems.js | 1 - test/transactWriteItem.js | 166 +++++++++++++++++++++++++++++++++- 2 files changed, 165 insertions(+), 2 deletions(-) diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index 0680079..1900f32 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -1,5 +1,4 @@ var async = require('async'), - { applyAttributeUpdates, applyUpdateExpression, deepClone } = require('./updateItem'), db = require('../db') module.exports = function transactWriteItem(store, data, cb) { diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js index 9b98010..6c4a39e 100644 --- a/test/transactWriteItem.js +++ b/test/transactWriteItem.js @@ -194,6 +194,46 @@ describe('transactWriteItem', function() { assertTransactionCanceled(transaction, 'Transaction cancelled, please refer cancellation reasons for specific reasons', done) }) + it('should return TransactionCanceledException for puts and updates of the same item with put first', function (done) { + var transaction = { + TransactItems: [{ + Put: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }, { + Update: { + TableName: helpers.testHashTable, + Key: {a: {S: 'aaaaa'}}, + UpdateExpression: 'SET b = :b', + ExpressionAttributeValues: { + ':b': { + S: 'b' + } + } + } + }] + } + assertTransactionCanceled(transaction, 'Transaction cancelled, please refer cancellation reasons for specific reasons', done) + }) + + it('should return ValidationException for item too large', function(done) { + var key = {a: {S: helpers.randomString()}} + var expressionAttributeValues = { + ':b': {S: new Array(helpers.MAX_SIZE).join('a')}, + ':c': {N: new Array(38 + 1).join('1') + new Array(89).join('0')}, + } + assertValidation({TransactItems: [ + {Update: + {TableName: helpers.testHashTable, + Key: key, + UpdateExpression: 'SET b = :b, c = :c', + ExpressionAttributeValues: expressionAttributeValues + }}]}, + 'Item size to update has exceeded the maximum allowed size', + done) + }) + it('should return ValidationException for key type mismatch in Put Item', function (done) { async.forEach([ {NULL: true}, @@ -274,7 +314,7 @@ describe('transactWriteItem', function() { }) }) - it('should write multiple items', function(done) { + it('should put multiple items', function(done) { var item = { a: {S: helpers.randomString()}, c: {S: 'c'}}, @@ -301,6 +341,130 @@ describe('transactWriteItem', function() { }) }) + it('should update multiple items', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Update: { + TableName: helpers.testHashTable, + Key: { + a: item.a + }, + UpdateExpression: 'SET c=:d', + ExpressionAttributeValues: { + ':d': { + S: 'd' + } + } + } + }, + { + Update: { + TableName: helpers.testHashTable, + Key: { + a: item2.a + }, + UpdateExpression: 'SET c=:d', + ExpressionAttributeValues: { + ':d': { + S: 'd' + } + } + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // update item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.eql({...item, c: {S: 'd'}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { + // put item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.eql({...item2, c: {S: 'd'}}) + done() + }) + }) + }) + }) + }) + }) + + it('should delete multiple items', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Delete: { + TableName: helpers.testHashTable, + Key: { + a: item.a + } + } + }, + { + Delete: { + TableName: helpers.testHashTable, + Key: { + a: item2.a + } + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // update item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item2.a}, ConsistentRead: true}), function(err, res) { + // put item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({}) + done() + }) + }) + }) + }) + }) + }) + it('should write, update, and delete in one transaction', function(done) { var item = { a: {S: helpers.randomString()}, From c1d78140cf8886663cfb63c2c93d531f2cef2ec1 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Fri, 30 Jul 2021 10:33:53 -0400 Subject: [PATCH 07/12] implement ConditionCheck --- actions/transactWriteItems.js | 26 + package-lock.json | 1188 ----------------------------- test/transactWriteItem.js | 292 +++++++ validations/transactWriteItems.js | 27 +- 4 files changed, 338 insertions(+), 1195 deletions(-) delete mode 100644 package-lock.json diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index 1900f32..190b75f 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -187,6 +187,32 @@ module.exports = function transactWriteItem(store, data, cb) { batchOpts[tableName] = [operation] } + return cb() + }) + }) + }) + } else if (transactItem.ConditionCheck) { + tableName = transactItem.ConditionCheck.TableName + + store.getTable(tableName, function (err, table) { + if (err) return cb(err) + + let key = db.createKey(transactItem.ConditionCheck.Key, table) + if (seenKeys[key]) { + return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) + } + + seenKeys[key] = true + + var itemDb = store.getItemDb(tableName) + + itemDb.lock(key, function(release) { + releaseLocks.push(release) + itemDb.get(key, function(err, oldItem) { + if (err && err.name != 'NotFoundError') return cb(err) + + if ((err = db.checkConditional(transactItem.ConditionCheck, oldItem)) != null) return cb(err) + return cb() }) }) diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 7b2a457..0000000 --- a/package-lock.json +++ /dev/null @@ -1,1188 +0,0 @@ -{ - "name": "dynalite", - "version": "3.2.1", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "abstract-leveldown": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.1.1.tgz", - "integrity": "sha1-9EutWGLXHHtBgRDXaYrCW+3yQ5Y=", - "optional": true, - "requires": { - "level-concat-iterator": "~2.0.0", - "xtend": "~4.0.0" - } - }, - "ansi-colors": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", - "integrity": "sha512-LEHHyuhlPY3TmuUYMh2oz89lTShfvgbmzaBcxve9t/9Wuy7Dwf4yoAKcND7KFT1HAQfqZ12qtc+DUrBMeKF9nw==", - "dev": true - }, - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", - "dev": true - }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha1-1yYl4jRKNlbjo61Pp0n6gymdgv8=", - "requires": { - "lodash": "^4.17.14" - } - }, - "aws4": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", - "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==", - "dev": true - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true - }, - "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" - }, - "big.js": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", - "integrity": "sha1-ZfCvOC9Xi83HQr2cKB6cstd2gyg=" - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "browser-stdout": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", - "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", - "dev": true - }, - "buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" - }, - "camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "dependencies": { - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, - "cliui": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", - "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", - "dev": true, - "requires": { - "string-width": "^3.1.0", - "strip-ansi": "^5.2.0", - "wrap-ansi": "^5.1.0" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } - } - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true - }, - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", - "dev": true - }, - "deferred-leveldown": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", - "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", - "requires": { - "abstract-leveldown": "~6.2.1", - "inherits": "^2.0.3" - }, - "dependencies": { - "abstract-leveldown": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", - "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - } - } - }, - "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, - "requires": { - "object-keys": "^1.0.12" - } - }, - "defined": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/defined/-/defined-0.0.0.tgz", - "integrity": "sha1-817qfXBekzuvE7LwOz+D2SFAOz4=" - }, - "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", - "dev": true - }, - "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, - "encoding-down": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", - "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", - "requires": { - "abstract-leveldown": "^6.2.1", - "inherits": "^2.0.3", - "level-codec": "^9.0.0", - "level-errors": "^2.0.0" - }, - "dependencies": { - "abstract-leveldown": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz", - "integrity": "sha512-TU5nlYgta8YrBMNpc9FwQzRbiXsj49gsALsXadbGHt9CROPzX5fB0rWDR5mtdpOOKa5XqRFpbj1QroPAoPzVjQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - } - } - }, - "errno": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", - "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", - "requires": { - "prr": "~1.0.1" - } - }, - "es-abstract": { - "version": "1.17.5", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", - "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", - "dev": true, - "requires": { - "es-to-primitive": "^1.2.1", - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.1", - "is-callable": "^1.1.5", - "is-regex": "^1.0.5", - "object-inspect": "^1.7.0", - "object-keys": "^1.1.1", - "object.assign": "^4.1.0", - "string.prototype.trimleft": "^2.1.1", - "string.prototype.trimright": "^2.1.1" - } - }, - "es-to-primitive": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dev": true, - "requires": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" - } - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "dev": true - }, - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true - }, - "find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", - "dev": true, - "requires": { - "locate-path": "^3.0.0" - } - }, - "flat": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/flat/-/flat-4.1.0.tgz", - "integrity": "sha512-Px/TiLIznH7gEDlPXcUD4KnBusa6kR6ayRUVcnEAbreRIuhkqow/mun59BuRXwoYk7ZQOLW1ZM05ilIvK38hFw==", - "dev": true, - "requires": { - "is-buffer": "~2.0.3" - } - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true - }, - "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true - }, - "functional-red-black-tree": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=" - }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true - }, - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true - }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, - "requires": { - "function-bind": "^1.1.1" - } - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "dev": true - }, - "has-symbols": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", - "dev": true - }, - "he": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, - "immediate": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.2.3.tgz", - "integrity": "sha1-0UD6j2FGWb1lQSMwl92qwlzdmRw=" - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "is-buffer": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", - "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==", - "dev": true - }, - "is-callable": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", - "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", - "dev": true - }, - "is-date-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", - "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true - }, - "is-regex": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", - "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", - "dev": true, - "requires": { - "has": "^1.0.3" - } - }, - "is-symbol": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", - "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", - "dev": true, - "requires": { - "has-symbols": "^1.0.1" - } - }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true - }, - "js-yaml": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", - "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", - "dev": true, - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, - "lazy": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/lazy/-/lazy-1.0.11.tgz", - "integrity": "sha1-2qBoIGKCVCwIgojpdcKXwa53tpA=" - }, - "level-codec": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.1.tgz", - "integrity": "sha512-ajFP0kJ+nyq4i6kptSM+mAvJKLOg1X5FiFPtLG9M5gCEZyBmgDi3FkDrvlMkEzrUn1cWxtvVmrvoS4ASyO/q+Q==" - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha1-HRAJzxCDQCUss4xR+XJzERk+YmM=" - }, - "level-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", - "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", - "requires": { - "errno": "~0.1.1" - } - }, - "level-iterator-stream": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", - "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", - "requires": { - "inherits": "^2.0.4", - "readable-stream": "^3.4.0", - "xtend": "^4.0.2" - }, - "dependencies": { - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - } - } - }, - "level-option-wrap": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/level-option-wrap/-/level-option-wrap-1.1.0.tgz", - "integrity": "sha1-rSDmjZ88IsiJdTHMaqevWWse0Sk=", - "requires": { - "defined": "~0.0.0" - } - }, - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - }, - "dependencies": { - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - } - } - }, - "leveldown": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.2.1.tgz", - "integrity": "sha1-6twhhKtMg/4NdIUzhP6Z3AoN5r4=", - "optional": true, - "requires": { - "abstract-leveldown": "~6.1.1", - "napi-macros": "~2.0.0", - "node-gyp-build": "~4.1.0" - } - }, - "levelup": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", - "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", - "requires": { - "deferred-leveldown": "~5.3.0", - "level-errors": "~2.0.0", - "level-iterator-stream": "~4.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - }, - "locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", - "dev": true, - "requires": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" - } - }, - "lock": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/lock/-/lock-1.1.0.tgz", - "integrity": "sha1-UxV0mdFlOxNspmRRBx/KYVcD+lU=" - }, - "lodash": { - "version": "4.17.19", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" - }, - "log-symbols": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", - "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==", - "dev": true, - "requires": { - "chalk": "^2.0.1" - } - }, - "ltgt": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", - "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" - }, - "memdown": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/memdown/-/memdown-5.1.0.tgz", - "integrity": "sha512-B3J+UizMRAlEArDjWHTMmadet+UKwHd3UjMgGBkZcKAxAYVPS9o0Yeiha4qvz7iGiL2Sb3igUft6p7nbFWctpw==", - "requires": { - "abstract-leveldown": "~6.2.1", - "functional-red-black-tree": "~1.0.1", - "immediate": "~3.2.3", - "inherits": "~2.0.1", - "ltgt": "~2.2.0", - "safe-buffer": "~5.2.0" - }, - "dependencies": { - "abstract-leveldown": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.2.tgz", - "integrity": "sha512-/a+Iwj0rn//CX0EJOasNyZJd2o8xur8Ce9C57Sznti/Ilt/cb6Qd8/k98A4ZOklXgTG+iAYYUs1OTG0s1eH+zQ==", - "requires": { - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - } - } - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" - }, - "mkdirp": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.4.tgz", - "integrity": "sha512-iG9AK/dJLtJ0XNgTuDbSyNS3zECqDlAhnQW4CsNxBG3LQJBbHmRX1egw39DmtOdCAqY+dKXV+sgPgilNWUKMVw==", - "dev": true, - "requires": { - "minimist": "^1.2.5" - } - }, - "mocha": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-6.2.3.tgz", - "integrity": "sha512-0R/3FvjIGH3eEuG17ccFPk117XL2rWxatr81a57D+r/x2uTYZRbdZ4oVidEUMh2W2TJDa7MdAb12Lm2/qrKajg==", - "dev": true, - "requires": { - "ansi-colors": "3.2.3", - "browser-stdout": "1.3.1", - "debug": "3.2.6", - "diff": "3.5.0", - "escape-string-regexp": "1.0.5", - "find-up": "3.0.0", - "glob": "7.1.3", - "growl": "1.10.5", - "he": "1.2.0", - "js-yaml": "3.13.1", - "log-symbols": "2.2.0", - "minimatch": "3.0.4", - "mkdirp": "0.5.4", - "ms": "2.1.1", - "node-environment-flags": "1.0.5", - "object.assign": "4.1.0", - "strip-json-comments": "2.0.1", - "supports-color": "6.0.0", - "which": "1.3.1", - "wide-align": "1.1.3", - "yargs": "13.3.2", - "yargs-parser": "13.1.2", - "yargs-unparser": "1.6.0" - } - }, - "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", - "dev": true - }, - "napi-macros": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", - "integrity": "sha1-K2uuQh57lutoeqbHenhYZAZwABs=", - "optional": true - }, - "node-environment-flags": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.5.tgz", - "integrity": "sha512-VNYPRfGfmZLx0Ye20jWzHUjyTW/c+6Wq+iLhDzUI4XmhrDd9l/FozXV3F2xOaXjvp0co0+v1YSR3CMP6g+VvLQ==", - "dev": true, - "requires": { - "object.getownpropertydescriptors": "^2.0.3", - "semver": "^5.7.0" - } - }, - "node-gyp-build": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", - "integrity": "sha1-1ycLXYZxcGjRFMxX//NS+W10X+s=", - "optional": true - }, - "object-inspect": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", - "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", - "dev": true - }, - "object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true - }, - "object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", - "dev": true, - "requires": { - "define-properties": "^1.1.2", - "function-bind": "^1.1.1", - "has-symbols": "^1.0.0", - "object-keys": "^1.0.11" - } - }, - "object.getownpropertydescriptors": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz", - "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1" - } - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "p-limit": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", - "integrity": "sha512-WGR+xHecKTr7EbUEhyLSh5Dube9JtdiG78ufaeLxTgpudf/20KqyMioIUZJAezlTIi6evxuoUs9YXc11cU+yzQ==", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", - "dev": true, - "requires": { - "p-limit": "^2.0.0" - } - }, - "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true - }, - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true - }, - "pegjs": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/pegjs/-/pegjs-0.10.0.tgz", - "integrity": "sha1-z4uvrm7d/0tafvsYUmnqr0YQ3b0=", - "dev": true - }, - "prr": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" - }, - "reachdown": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reachdown/-/reachdown-1.1.0.tgz", - "integrity": "sha512-6LsdRe4cZyOjw4NnvbhUd/rGG7WQ9HMopPr+kyL018Uci4kijtxcGR5kVb5Ln13k4PEE+fEFQbjfOvNw7cnXmA==" - }, - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "dev": true - }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", - "dev": true - }, - "safe-buffer": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", - "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", - "dev": true - }, - "should": { - "version": "13.2.3", - "resolved": "https://registry.npmjs.org/should/-/should-13.2.3.tgz", - "integrity": "sha1-ltjlrPPpe0nYm1H+qlro0H71jxA=", - "dev": true, - "requires": { - "should-equal": "^2.0.0", - "should-format": "^3.0.3", - "should-type": "^1.4.0", - "should-type-adaptors": "^1.0.1", - "should-util": "^1.0.0" - } - }, - "should-equal": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/should-equal/-/should-equal-2.0.0.tgz", - "integrity": "sha1-YHLPgwRzYIZ+aOmLCdcRQ9BO4MM=", - "dev": true, - "requires": { - "should-type": "^1.4.0" - } - }, - "should-format": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/should-format/-/should-format-3.0.3.tgz", - "integrity": "sha1-m/yPdPo5IFxT04w01xcwPidxJPE=", - "dev": true, - "requires": { - "should-type": "^1.3.0", - "should-type-adaptors": "^1.0.1" - } - }, - "should-type": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/should-type/-/should-type-1.4.0.tgz", - "integrity": "sha1-B1bYzoRt/QmEOmlHcZ36DUz/XPM=", - "dev": true - }, - "should-type-adaptors": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/should-type-adaptors/-/should-type-adaptors-1.1.0.tgz", - "integrity": "sha1-QB5/M7VTMDOUTVzYvytlAneS4no=", - "dev": true, - "requires": { - "should-type": "^1.3.0", - "should-util": "^1.0.0" - } - }, - "should-util": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/should-util/-/should-util-1.0.0.tgz", - "integrity": "sha1-yYzaN0qmsZDfi6h8mInCtNtiAGM=", - "dev": true - }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", - "dev": true - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - } - }, - "string.prototype.trimend": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.0.tgz", - "integrity": "sha512-EEJnGqa/xNfIg05SxiPSqRS7S9qwDhYts1TSLR1BQfYUfPe1stofgGKvwERK9+9yf+PpfBMlpBaCHucXGPQfUA==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" - } - }, - "string.prototype.trimleft": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", - "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimstart": "^1.0.0" - } - }, - "string.prototype.trimright": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", - "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimend": "^1.0.0" - } - }, - "string.prototype.trimstart": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.0.tgz", - "integrity": "sha512-iCP8g01NFYiiBOnwG1Xc3WZLyoo+RuBymwIlWncShXDDJYWN6DbnM3odslBJdgCdRlq94B5s63NWAZlcn2CS4w==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" - } - }, - "string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "requires": { - "safe-buffer": "~5.2.0" - } - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "dev": true, - "requires": { - "ansi-regex": "^3.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", - "dev": true - }, - "subleveldown": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/subleveldown/-/subleveldown-5.0.0.tgz", - "integrity": "sha512-DotbiAIyOWSsidM06/m+EsBHXRyP7EgPlDDD5GVn6JcoDFcVZbp+VN9bOdErNtDWwgR+lJDuKwcJ8nKqSq9Ixg==", - "requires": { - "abstract-leveldown": "^6.2.3", - "encoding-down": "^6.2.0", - "inherits": "^2.0.3", - "level-option-wrap": "^1.1.0", - "levelup": "^4.3.1", - "reachdown": "^1.1.0" - }, - "dependencies": { - "abstract-leveldown": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz", - "integrity": "sha512-TU5nlYgta8YrBMNpc9FwQzRbiXsj49gsALsXadbGHt9CROPzX5fB0rWDR5mtdpOOKa5XqRFpbj1QroPAoPzVjQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - } - } - }, - "supports-color": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.0.0.tgz", - "integrity": "sha512-on9Kwidc1IUQo+bQdhi8+Tijpo0e1SS6RoGo2guUwn5vdaxw8RXOF9Vb2ws+ihWOmh4JnCJOvaziZWP1VABaLg==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dev": true, - "requires": { - "isexe": "^2.0.0" - } - }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", - "dev": true - }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "dev": true, - "requires": { - "string-width": "^1.0.2 || 2" - } - }, - "wrap-ansi": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", - "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.0", - "string-width": "^3.0.0", - "strip-ansi": "^5.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } - } - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, - "y18n": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", - "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", - "dev": true - }, - "yargs": { - "version": "13.3.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", - "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", - "dev": true, - "requires": { - "cliui": "^5.0.0", - "find-up": "^3.0.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^3.0.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^13.1.2" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } - } - } - }, - "yargs-parser": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", - "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } - }, - "yargs-unparser": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-1.6.0.tgz", - "integrity": "sha512-W9tKgmSn0DpSatfri0nx52Joq5hVXgeLiqR/5G0sZNDoLZFOr/xjBUDcShCOGNsBnEMNo1KAMBkTej1Hm62HTw==", - "dev": true, - "requires": { - "flat": "^4.1.0", - "lodash": "^4.17.15", - "yargs": "^13.3.0" - } - } - } -} diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js index 6c4a39e..6e85eff 100644 --- a/test/transactWriteItem.js +++ b/test/transactWriteItem.js @@ -108,6 +108,13 @@ describe('transactWriteItem', function() { 'Member must not be null', done) }) + it('should return ValidationException for invalid ConditionCheck request in TransactItems', function (done) { + assertValidation({TransactItems: [{ConditionCheck: {}}]}, + '1 validation error detected: ' + + 'Value null at \'transactItems.1.member.conditionCheck.key\' failed to satisfy constraint: ' + + 'Member must not be null', done) + }) + it('should return ValidationException for invalid metadata and missing requests', function (done) { assertValidation({TransactItems: [], ReturnConsumedCapacity: 'hi', ReturnItemCollectionMetrics: 'hi'}, [ 'Value \'hi\' at \'returnConsumedCapacity\' failed to satisfy constraint: ' + @@ -217,6 +224,24 @@ describe('transactWriteItem', function() { assertTransactionCanceled(transaction, 'Transaction cancelled, please refer cancellation reasons for specific reasons', done) }) + it('should return TransactionCanceledException for puts and condition checks of the same item with put first', function (done) { + var transaction = { + TransactItems: [{ + Put: { + TableName: helpers.testHashTable, + Item: {a: {S: 'aaaaa'}} + } + }, { + ConditionCheck: { + TableName: helpers.testHashTable, + Key: {a: {S: 'aaaaa'}}, + ConditionExpression: 'attribute_exists(a)' + } + }] + } + assertTransactionCanceled(transaction, 'Transaction cancelled, please refer cancellation reasons for specific reasons', done) + }) + it('should return ValidationException for item too large', function(done) { var key = {a: {S: helpers.randomString()}} var expressionAttributeValues = { @@ -656,6 +681,173 @@ describe('transactWriteItem', function() { }) }) + it('should fail to write with failed ConditionCheck in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: item + } + }, + { + ConditionCheck: { + TableName: helpers.testHashTable, + Key: { + a: item2.a + }, + ConditionExpression: 'attribute_not_exists(c)' + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(400) + res.body.message.should.equal('The conditional request failed') + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({}) + done() + }) + }) + }) + }) + + it('should succeed to write with ConditionCheck in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: item + } + }, + { + ConditionCheck: { + TableName: helpers.testHashTable, + Key: { + a: item2.a + }, + ConditionExpression: 'attribute_not_exists(c)' + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(400) + res.body.message.should.equal('The conditional request failed') + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({}) + done() + }) + }) + }) + }) + + it('should succeed to write in table A with succeeded ConditionCheck in table B in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = {a: {S: helpers.randomString()}, b: {S: helpers.randomString()}, g: {N: '23'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: item + } + }, + { + ConditionCheck: { + TableName: helpers.testRangeTable, + Key: {a: item2.a, b: item2.b}, + ConditionExpression: 'attribute_not_exists(c)' + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testRangeTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.should.eql(item) + done() + }) + }) + }) + }) + + it('should fail to write in table A with failed ConditionCheck in table B in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + item2 = {a: {S: helpers.randomString()}, b: {S: helpers.randomString()}, g: {N: '23'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: item + } + }, + { + ConditionCheck: { + TableName: helpers.testRangeTable, + Key: {a: item2.a, b: item2.b}, + ConditionExpression: 'attribute_not_exists(g)' + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testRangeTable, Item: item2}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(400) + res.body.message.should.equal('The conditional request failed') + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({}) + done() + }) + }) + }) + }) + it('should write to two different tables', function(done) { var hashItem = {a: {S: helpers.randomString()}, c: {S: 'c'}}, rangeItem = {a: {S: helpers.randomString()}, b: {S: helpers.randomString()}, g: {N: '23'}} @@ -762,6 +954,106 @@ describe('transactWriteItem', function() { }) }) }) + + it('should complete successfully with updates and atomic counter decrement', function (done) { + var atomicCounter = { + a: { + S: 'atomicCounter' + }, + counter: { + N: '1' + } + } + + var item = { + a: { + S: 'item' + }, + b: { + M: { + 'one': { + S: 'itemname', + }, + 'two': { + N: '123456' + } + } + } + } + + var transaction = { + TransactItems: [{ + Update: { + TableName: helpers.testHashTable, + Key: { + a: atomicCounter.a + }, + UpdateExpression: `SET #counter = if_not_exists(#counter, :zero) + :increment`, + ExpressionAttributeNames: { + '#counter': 'counter' + }, + ExpressionAttributeValues: { + ':increment': { + N: '-1' + }, + ':zero': { + N: '0' + } + } + } + }, + { + Update: { + TableName: helpers.testHashTable, + Key: {a: item.a}, + UpdateExpression: 'SET b = :b', + ExpressionAttributeValues: { + ':b': { + M: { + 'one': { + S: 'itemname', + }, + 'two': { + N: '654321' + } + } + } + } + } + } + ] + } + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: atomicCounter}), function(err, res) { + if (err) return done(err) + res.statusCode.should.eql(200) + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transaction), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.b.M.should.eql({'one': { + S: 'itemname', + }, + 'two': { + N: '654321' + }}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: atomicCounter.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.counter.should.eql({N: '0'}) + done() + }) + }) + }) + }) + }) + }) }) }) }) \ No newline at end of file diff --git a/validations/transactWriteItems.js b/validations/transactWriteItems.js index b66ccc3..a62fcf7 100644 --- a/validations/transactWriteItems.js +++ b/validations/transactWriteItems.js @@ -142,30 +142,43 @@ exports.types = { } exports.custom = function(data, store) { - var i, request, msg + var i, request, msg, key for (i = 0; i < data.TransactItems.length; i++) { request = data.TransactItems[i] if (request.Put) { - for (let key in request.Put.Item) { + for (key in request.Put.Item) { msg = validations.validateAttributeValue(request.Put.Item[key]) if (msg) return msg } if (db.itemSize(request.Put.Item) > store.options.maxItemSize) return 'Item size has exceeded the maximum allowed size' } else if (request.Delete) { - msg = validations.validateAttributeValue(request.Delete.Key) - if (msg) return msg + for (key in request.Delete.Key) { + msg = validations.validateAttributeValue(request.Delete.Key[key]) + if (msg) return msg + } } else if (request.Update) { - var msg = validations.validateExpressionParams(request.Update, + for (key in request.Update.Key) { + msg = validations.validateAttributeValue(request.Update.Key[key]) + if (msg) return msg + } + msg = validations.validateExpressionParams(request.Update, ['UpdateExpression', 'ConditionExpression'], ['AttributeUpdates', 'Expected']) if (msg) return msg - msg = validations.validateAttributeValue(request.Update.Key) - if (msg) return msg msg = validations.validateAttributeConditions(request.Update) if (msg) return msg msg = validations.validateExpressions(request.Update) if (msg) return msg + } else if (request.ConditionCheck) { + for (key in request.ConditionCheck.Key) { + msg = validations.validateAttributeValue(request.ConditionCheck.Key[key]) + if (msg) return msg + } + msg = validations.validateExpressionParams(request.ConditionCheck, ['ConditionExpression'], []) + if (msg) return msg + msg = validations.validateExpressions(request.ConditionCheck) + if (msg) return msg } else { return 'The action or operation requested is invalid. Verify that the action is typed correctly.' } From dd2e82959f06101485af4031a4e859dcc4c5ad0a Mon Sep 17 00:00:00 2001 From: n0286293 Date: Fri, 30 Jul 2021 12:00:21 -0400 Subject: [PATCH 08/12] actually release item locks after transaction completes --- actions/transactWriteItems.js | 2 +- test/transactWriteItem.js | 58 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index 190b75f..beb93e0 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -25,7 +25,7 @@ module.exports = function transactWriteItem(store, data, cb) { callback(results) }) }, function(err) { - async.parallel(releaseLocks) + releaseLocks.forEach(release => release()()) if (err) return cb(err) var res = {UnprocessedItems: {}}, tableUnits = {} diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js index 6e85eff..7547bf9 100644 --- a/test/transactWriteItem.js +++ b/test/transactWriteItem.js @@ -1054,6 +1054,64 @@ describe('transactWriteItem', function() { }) }) }) + + it('should update the same row multiple times', function (done) { + var atomicCounter = { + a: { + S: 'atomicCounter' + }, + counter: { + N: '1' + } + } + + var transaction = { + TransactItems: [{ + Update: { + TableName: helpers.testHashTable, + Key: { + a: atomicCounter.a + }, + UpdateExpression: `SET #counter = if_not_exists(#counter, :zero) + :increment`, + ExpressionAttributeNames: { + '#counter': 'counter' + }, + ExpressionAttributeValues: { + ':increment': { + N: '1' + }, + ':zero': { + N: '0' + } + } + } + } + ] + } + + request(opts(transaction), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: atomicCounter.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.counter.N.should.eql('1') + request(opts(transaction), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({UnprocessedItems: {}}) + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: atomicCounter.a}, ConsistentRead: true}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Item.counter.N.should.eql('2') + done() + }) + }) + }) + }) + }) + }) }) }) \ No newline at end of file From e8149c2db0f9a2da98e82041d7700759d8a88085 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Fri, 6 Aug 2021 10:24:13 -0400 Subject: [PATCH 09/12] update indexes and test that that actually happens --- actions/transactWriteItems.js | 38 +++++++++- test/transactWriteItem.js | 129 ++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index beb93e0..3b3e671 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -5,6 +5,7 @@ module.exports = function transactWriteItem(store, data, cb) { var seenKeys = {} var batchOpts = {} var releaseLocks = [] + var indexUpdates = {} async.eachSeries(data.TransactItems, addActions, function (err) { if (err) { @@ -20,9 +21,17 @@ module.exports = function transactWriteItem(store, data, cb) { async.each(operationsbyTable, function([tableName, ops], callback) { var itemDb = store.getItemDb(tableName) - itemDb.batch(ops, function(err, results) { - if (err) callback(err) - callback(results) + + store.getTable(tableName, function(err, table) { + indexUpdates[tableName].forEach(update => { + db.updateIndexes(store, table, update.existingItem, update.item, function(err) { + if (err) return callback(err) + }) + }) + itemDb.batch(ops, function(err, results) { + if (err) callback(err) + callback(results) + }) }) }, function(err) { releaseLocks.forEach(release => release()()) @@ -92,10 +101,17 @@ module.exports = function transactWriteItem(store, data, cb) { value } + let indexUpdate = { + existingItem: oldItem, + item: value + } + if (batchOpts[tableName]) { batchOpts[tableName].push(operation) + indexUpdates[tableName].push(indexUpdate) } else { batchOpts[tableName] = [operation] + indexUpdates[tableName] = [indexUpdate] } return cb() @@ -129,10 +145,16 @@ module.exports = function transactWriteItem(store, data, cb) { key } + let indexUpdate = { + existingItem: oldItem + } + if (batchOpts[tableName]) { batchOpts[tableName].push(operation) + indexUpdates[tableName].push(indexUpdate) } else { batchOpts[tableName] = [operation] + indexUpdates[tableName] = [indexUpdate] } return cb() }) @@ -147,6 +169,7 @@ module.exports = function transactWriteItem(store, data, cb) { if ((err = db.validateUpdates(transactItem.Update.AttributeUpdates, transactItem.Update._updates, table)) != null) return cb(err) let key = db.createKey(transactItem.Update.Key, table) + console.log(key) if (seenKeys[key]) { return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) } @@ -163,9 +186,11 @@ module.exports = function transactWriteItem(store, data, cb) { if ((err = db.checkConditional(transactItem.Update, oldItem)) != null) return cb(err) var item = transactItem.Update.Key + console.log('old item', oldItem) if (oldItem) { item = db.deepClone(oldItem) + console.log('did deep clone') } err = transactItem.Update._updates ? db.applyUpdateExpression(transactItem.Update._updates.sections, table, item) : @@ -181,10 +206,17 @@ module.exports = function transactWriteItem(store, data, cb) { value: item } + let indexUpdate = { + existingItem: oldItem, + item: item + } + if (batchOpts[tableName]) { batchOpts[tableName].push(operation) + indexUpdates[tableName].push(indexUpdate) } else { batchOpts[tableName] = [operation] + indexUpdates[tableName] = [indexUpdate] } return cb() diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js index 7547bf9..aa3aee3 100644 --- a/test/transactWriteItem.js +++ b/test/transactWriteItem.js @@ -1112,6 +1112,135 @@ describe('transactWriteItem', function() { }) }) + it('should update the index', function (done) { + var key = {a: {S: helpers.randomString()}, b: {S: helpers.randomString()}} + var transaction = { + TransactItems: [{ + Update: { + TableName: helpers.testRangeTable, + Key: key, + UpdateExpression: 'set c = a, d = b, e = a, f = b' + } + } + ] + } + request(opts(transaction), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + ConsistentRead: true, + IndexName: 'index1', + KeyConditions: { + a: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + c: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([{a: key.a, b: key.b, c: key.a, d: key.b, e: key.a, f: key.b}]) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + ConsistentRead: true, + IndexName: 'index2', + KeyConditions: { + a: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + d: {ComparisonOperator: 'EQ', AttributeValueList: [key.b]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([{a: key.a, b: key.b, c: key.a, d: key.b}]) + transaction.TransactItems[0].Update = { + TableName: helpers.testRangeTable, + Key: key, + UpdateExpression: 'set c = b, d = a, e = b, f = a', + } + request(opts(transaction), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + ConsistentRead: true, + IndexName: 'index1', + KeyConditions: { + a: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + c: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([]) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + ConsistentRead: true, + IndexName: 'index2', + KeyConditions: { + a: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + d: {ComparisonOperator: 'EQ', AttributeValueList: [key.b]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([]) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + ConsistentRead: true, + IndexName: 'index1', + KeyConditions: { + a: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + c: {ComparisonOperator: 'EQ', AttributeValueList: [key.b]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([{a: key.a, b: key.b, c: key.b, d: key.a, e: key.b, f: key.a}]) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + ConsistentRead: true, + IndexName: 'index2', + KeyConditions: { + a: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + d: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([{a: key.a, b: key.b, c: key.b, d: key.a}]) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + IndexName: 'index3', + KeyConditions: { + c: {ComparisonOperator: 'EQ', AttributeValueList: [key.b]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([{a: key.a, b: key.b, c: key.b, d: key.a, e: key.b, f: key.a}]) + request(helpers.opts('Query', { + TableName: helpers.testRangeTable, + IndexName: 'index4', + KeyConditions: { + c: {ComparisonOperator: 'EQ', AttributeValueList: [key.b]}, + d: {ComparisonOperator: 'EQ', AttributeValueList: [key.a]}, + }, + }), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.Items.should.eql([{a: key.a, b: key.b, c: key.b, d: key.a, e: key.b}]) + done() + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) + }) }) }) \ No newline at end of file From d427708ee750a06ee09cbecd6319fd9f28318912 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Fri, 6 Aug 2021 10:56:59 -0400 Subject: [PATCH 10/12] remove console log debug statements --- actions/transactWriteItems.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/actions/transactWriteItems.js b/actions/transactWriteItems.js index 3b3e671..921205b 100644 --- a/actions/transactWriteItems.js +++ b/actions/transactWriteItems.js @@ -169,7 +169,6 @@ module.exports = function transactWriteItem(store, data, cb) { if ((err = db.validateUpdates(transactItem.Update.AttributeUpdates, transactItem.Update._updates, table)) != null) return cb(err) let key = db.createKey(transactItem.Update.Key, table) - console.log(key) if (seenKeys[key]) { return cb(db.transactionCancelledException('Transaction cancelled, please refer cancellation reasons for specific reasons')) } @@ -186,11 +185,9 @@ module.exports = function transactWriteItem(store, data, cb) { if ((err = db.checkConditional(transactItem.Update, oldItem)) != null) return cb(err) var item = transactItem.Update.Key - console.log('old item', oldItem) if (oldItem) { item = db.deepClone(oldItem) - console.log('did deep clone') } err = transactItem.Update._updates ? db.applyUpdateExpression(transactItem.Update._updates.sections, table, item) : From cfc381cbad87c93915be55ede0e8f38f8bcd38d4 Mon Sep 17 00:00:00 2001 From: n0286293 Date: Fri, 12 Nov 2021 13:20:00 -0500 Subject: [PATCH 11/12] add new line to end of db/index.js file --- db/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/index.js b/db/index.js index a5cd57c..9765f1b 100644 --- a/db/index.js +++ b/db/index.js @@ -1163,4 +1163,4 @@ function addToParent(parent, attr, val, toSquash) { } parent.L[attr] = val } -} \ No newline at end of file +} From 53e4733264b9709beda386871dc8aac36511391c Mon Sep 17 00:00:00 2001 From: Jacqueline Peterschmidt-Gargas Date: Mon, 14 Nov 2022 14:53:58 -0500 Subject: [PATCH 12/12] fix transactWriteItems ignoring condition expressions in Puts and Deletes --- test/transactWriteItem.js | 37 +++++++++++++++++++++++++++++++ validations/transactWriteItems.js | 4 ++++ 2 files changed, 41 insertions(+) diff --git a/test/transactWriteItem.js b/test/transactWriteItem.js index aa3aee3..be4efe3 100644 --- a/test/transactWriteItem.js +++ b/test/transactWriteItem.js @@ -681,6 +681,43 @@ describe('transactWriteItem', function() { }) }) + it('should fail to put with failed condition expression in one transaction', function(done) { + var item = { + a: {S: helpers.randomString()}, + c: {S: 'c'}}, + transactReq = {TransactItems: []} + + transactReq.TransactItems = [ + { + Put: { + TableName: helpers.testHashTable, + Item: { + ...item, + c: {S: 'd'} + }, + ConditionExpression: 'attribute_not_exists(c)', + } + } + ] + + request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(200) + request(opts(transactReq), function(err, res) { + if (err) return done(err) + res.statusCode.should.equal(400) + res.body.message.should.equal('The conditional request failed') + request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: {a: item.a}, ConsistentRead: true}), function(err, res) { + // update item + if (err) return done(err) + res.statusCode.should.equal(200) + res.body.should.eql({Item: item}) + done() + }) + }) + }) + }) + it('should fail to write with failed ConditionCheck in one transaction', function(done) { var item = { a: {S: helpers.randomString()}, diff --git a/validations/transactWriteItems.js b/validations/transactWriteItems.js index a62fcf7..a0c0efa 100644 --- a/validations/transactWriteItems.js +++ b/validations/transactWriteItems.js @@ -152,11 +152,15 @@ exports.custom = function(data, store) { } if (db.itemSize(request.Put.Item) > store.options.maxItemSize) return 'Item size has exceeded the maximum allowed size' + msg = validations.validateExpressions(request.Put) + if (msg) return msg } else if (request.Delete) { for (key in request.Delete.Key) { msg = validations.validateAttributeValue(request.Delete.Key[key]) if (msg) return msg } + msg = validations.validateExpressions(request.Delete) + if (msg) return msg } else if (request.Update) { for (key in request.Update.Key) { msg = validations.validateAttributeValue(request.Update.Key[key])