From 57f5d8b49473c35c15f2ce47ca740e0d99a982ff Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 8 Jan 2021 15:48:17 -0500 Subject: [PATCH 1/3] node: rpc dumpzone Co-authored-by: James Stevens Co-authored-by: Mark Tyneway --- lib/node/rpc.js | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/lib/node/rpc.js b/lib/node/rpc.js index b892625aab..4882e432cb 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -6,6 +6,10 @@ 'use strict'; +const constants = require('bns/lib/constants'); +const {types} = constants; +const fs = require('fs'); +const bns = require('bns'); const assert = require('bsert'); const bweb = require('bweb'); const {Lock} = require('bmutex'); @@ -247,6 +251,7 @@ class RPC extends RPCBase { this.add('validateresource', this.validateResource); this.add('resetrootcache', this.resetRootCache); + this.add('dumpzone', this.dumpzone); // Compat // this.add('getnameinfo', this.getNameInfo); @@ -2577,6 +2582,68 @@ class RPC extends RPCBase { return null; } + async dumpzone(args, help) { + if (help || args.length !== 1) + throw new RPCError(errs.MISC_ERROR, 'dumpzone '); + + const valid = new Validator(args); + const filename = valid.str(0, null); + if (filename == null) + throw new RPCError(errs.MISC_ERROR, 'dumpzone '); + + const tmp = filename + '~'; + this.logger.debug(`dumping root zone to file: ${filename}`); + + let fd = null; + let fileErr = null; + fd = fs.createWriteStream(tmp, { flags: 'a+' }); + fd.on('error', (err) => { + fd.end(); + fd = null; + fileErr = err; + }); + + const tree = this.chain.db.tree; + const iter = tree.iterator(true); + + let count = 0; + while ((await iter.next()) && (fd != null)) { + count++; + if (count % 10000 === 0) + this.logger.debug('dumpzone names processed: %d', count); + + if (fd == null) + break; + + const {value} = iter; + const ns = NameState.decode(value); + + if (ns.data.length <= 0) + continue; + + const fqdn = bns.util.fqdn(ns.name.toString('ascii')); + const resource = Resource.decode(ns.data); + const zone = resource.toZone(fqdn); + for (const record of zone) { + if (fd == null) + break; + + if (record.type !== types.RRSIG) + fd.write(record.toString() + '\n'); + } + } + + if (fd == null) + return fileErr; + + fd.end(); + fs.renameSync(tmp, filename); + + this.logger.debug('root zone dump complete. Total names: %d', count); + + return filename; + } + /* * Helpers */ From 3b823fb7f07a8e6e1d5094b70f8076b57ef9f44a Mon Sep 17 00:00:00 2001 From: Bennett Hoffman Date: Wed, 20 Jan 2021 09:19:19 -0500 Subject: [PATCH 2/3] Modularize zippy's dumpzone, filter TXT --- lib/node/dumpzone.js | 50 +++++++++++++++++++++++++++++++++ lib/node/rpc.js | 66 +++++++++++--------------------------------- 2 files changed, 66 insertions(+), 50 deletions(-) create mode 100644 lib/node/dumpzone.js diff --git a/lib/node/dumpzone.js b/lib/node/dumpzone.js new file mode 100644 index 0000000000..34c35cf587 --- /dev/null +++ b/lib/node/dumpzone.js @@ -0,0 +1,50 @@ +'use strict'; + +const bns = require('bns'); +const {types} = require('bns/lib/constants'); +const stream = require('stream'); + +const NameState = require('../covenants/namestate'); +const {Resource} = require('../dns/resource'); + +/** + * @typedef {import('../blockchain').Chain} Chain + */ + +/** + * readableStream produces a newline-delimited list of all + * DNS records on a chain, except for RRSIG, as a utf8 + * encoded string. + * + * @param {Chain} chain the chain to stream from + * @returns {stream.Readable} a readable stream of DNS records + */ +function readableStream(chain) { + const iter = chain.db.tree.iterator(true); + + async function* gen() { + while (await iter.next()) { + /** @type {NameState} */ + const ns = NameState.decode(iter.value); + if (ns.data.length <= 0) + continue; + + /** @type {string} */ + const fqdn = bns.util.fqdn(ns.name.toString('ascii')); + + /** @type {Resource} */ + const resource = Resource.decode(ns.data); + const zone = resource.toZone(fqdn); + for (const record of zone) { + if (record.type !== types.RRSIG && record.type !== types.TXT) + yield record.toString() + '\n'; + } + } + } + + return stream.Readable.from(gen(), {encoding: 'utf8'}); +} + +module.exports = { + readableStream: readableStream +}; diff --git a/lib/node/rpc.js b/lib/node/rpc.js index 4882e432cb..a5af0b44de 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -6,10 +6,7 @@ 'use strict'; -const constants = require('bns/lib/constants'); -const {types} = constants; const fs = require('fs'); -const bns = require('bns'); const assert = require('bsert'); const bweb = require('bweb'); const {Lock} = require('bmutex'); @@ -46,6 +43,8 @@ const AirdropProof = require('../primitives/airdropproof'); const {EXP} = consensus; const RPCBase = bweb.RPC; const RPCError = bweb.RPCError; +const dumpzone = require('./dumpzone'); +const stream = require('stream'); /* * Constants @@ -2594,54 +2593,21 @@ class RPC extends RPCBase { const tmp = filename + '~'; this.logger.debug(`dumping root zone to file: ${filename}`); - let fd = null; - let fileErr = null; - fd = fs.createWriteStream(tmp, { flags: 'a+' }); - fd.on('error', (err) => { - fd.end(); - fd = null; - fileErr = err; + return new Promise((resolve, reject) => { + stream.pipeline( + dumpzone.readableStream(this.chain), + fs.createWriteStream(tmp, { flags: 'a+' }), + (err) => { + if (err) { + reject(err); + } else { + fs.renameSync(tmp, filename); + this.logger.debug('root zone dump complete'); + resolve(filename); + } + } + ); }); - - const tree = this.chain.db.tree; - const iter = tree.iterator(true); - - let count = 0; - while ((await iter.next()) && (fd != null)) { - count++; - if (count % 10000 === 0) - this.logger.debug('dumpzone names processed: %d', count); - - if (fd == null) - break; - - const {value} = iter; - const ns = NameState.decode(value); - - if (ns.data.length <= 0) - continue; - - const fqdn = bns.util.fqdn(ns.name.toString('ascii')); - const resource = Resource.decode(ns.data); - const zone = resource.toZone(fqdn); - for (const record of zone) { - if (fd == null) - break; - - if (record.type !== types.RRSIG) - fd.write(record.toString() + '\n'); - } - } - - if (fd == null) - return fileErr; - - fd.end(); - fs.renameSync(tmp, filename); - - this.logger.debug('root zone dump complete. Total names: %d', count); - - return filename; } /* From 4e4412638025d468d6c0f49601126069e7dad403 Mon Sep 17 00:00:00 2001 From: Bennett Hoffman Date: Thu, 21 Jan 2021 22:00:06 -0500 Subject: [PATCH 3/3] Upload zone dump to S3 --- lib/node/dumpzone.js | 7 +-- lib/node/fullnode.js | 6 ++- lib/node/http.js | 101 +++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 96 ++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- 5 files changed, 208 insertions(+), 5 deletions(-) diff --git a/lib/node/dumpzone.js b/lib/node/dumpzone.js index 34c35cf587..d55e743f96 100644 --- a/lib/node/dumpzone.js +++ b/lib/node/dumpzone.js @@ -36,13 +36,14 @@ function readableStream(chain) { const resource = Resource.decode(ns.data); const zone = resource.toZone(fqdn); for (const record of zone) { - if (record.type !== types.RRSIG && record.type !== types.TXT) - yield record.toString() + '\n'; + if (record.type !== types.RRSIG && record.type !== types.TXT) { + yield Buffer.from(record.toString() + '\n', 'utf8'); + } } } } - return stream.Readable.from(gen(), {encoding: 'utf8'}); + return stream.Readable.from(gen(), {objectMode: false}); } module.exports = { diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 5f24d87933..65f9cd321e 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -142,7 +142,11 @@ class FullNode extends Node { port: this.config.uint('http-port'), apiKey: this.config.str('api-key'), noAuth: this.config.bool('no-auth'), - cors: this.config.bool('cors') + cors: this.config.bool('cors'), + s3: { + bucket: this.config.str('s3-dump-bucket'), + key: this.config.str('s3-dump-key') + } }); this.ns = new RootServer({ diff --git a/lib/node/http.js b/lib/node/http.js index d0e266e659..195647a3a7 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -21,6 +21,8 @@ const Claim = require('../primitives/claim'); const Address = require('../primitives/address'); const Network = require('../protocol/network'); const pkg = require('../pkg'); +const AWS = require('aws-sdk'); +const dumpzone = require('./dumpzone'); /** * HTTP @@ -48,6 +50,11 @@ class HTTP extends Server { this.miner = this.node.miner; this.rpc = this.node.rpc; + this.s3 = null; + /** @type {AWS.S3.ManagedUpload | null} */ + this.runningUpload = null; + this.uploadProgress = null; + this.init(); } @@ -72,6 +79,22 @@ class HTTP extends Server { this.initRouter(); this.initSockets(); + this.initS3(); + } + + /** + * Initialize the S3 service, if possible. + * @private + */ + + initS3() { + AWS.config.getCredentials((err) => { + if (err) + this.logger.warning('couldn\'t load AWS credentials', err.stack); + // credentials not loaded + else + this.s3 = new AWS.S3(); + }); } /** @@ -452,6 +475,79 @@ class HTTP extends Server { res.json(200, { success: true }); }); + + // Initiate a zone dump to S3 + this.post('/dump-zone-to-s3', async (req, res) => { + if(this.s3 === null) { + res.json(501, + { + success: false, + message: 'AWS is not properly configured' + } + ); + return; + } + + if(this.runningUpload !== null) { + res.json(202, + { + success: true, + message: 'upload already in progress' + } + ); + return; + } + + this.runningUpload = this.s3.upload({ + Bucket: this.options.s3DumpConfig.bucket, + Key: this.options.s3DumpConfig.key, + Body: dumpzone.readableStream(this.chain) + }, (err, data) => { + // TODO - capture status, do a rename? + this.runningUpload = null; + this.uploadProgress = null; + }); + + this.runningUpload.on('httpUploadProgress', (progress) => { + this.uploadProgress = progress; + }); + + res.json(202, { + success: true, + message: 'upload started' + }); + return; + }); + + this.get('/dump-zone-to-s3', async (req, res) => { + if(this.s3 === null) { + res.json(501, + { + success: false, + message: 'AWS is not properly configured' + } + ); + return; + } + + if(this.runningUpload === null) { + res.json(200, + { + success: true, + message: 'No upload is currently running' + } + ); + return; + } + + res.json(200, + { + success: true, + message: 'Upload in progress', + progress: this.uploadProgress + } + ); + }); } /** @@ -894,6 +990,11 @@ class HTTPOptions { this.noAuth = true; } + if (options.s3 != null) { + assert(typeof options.s3 === 'object'); + this.s3DumpConfig = options.s3; + } + return this; } diff --git a/package-lock.json b/package-lock.json index e5f3f513a8..f7f47d360c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -204,6 +204,22 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "aws-sdk": { + "version": "2.829.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.829.0.tgz", + "integrity": "sha512-0LV0argbcE1HhOeCeCZWUbpP4rWzwqe+0WmnR+jCJPY0w0n/ntGU/GW8obzhhhUej8pS4AkuAJNgzbwlTnxUmw==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, "babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -224,6 +240,11 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "bcfg": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/bcfg/-/bcfg-0.1.6.tgz", @@ -421,6 +442,16 @@ "resolved": "https://registry.npmjs.org/budp/-/budp-0.1.6.tgz", "integrity": "sha512-o+a8NPq3DhV91j4nInjht2md6mbU1XL+7ciPltP66rw5uD3KP1m5r8lA94LZVaPKcFdJ0l2HVVzRNxnY26Pefg==" }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "buffer-map": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/buffer-map/-/buffer-map-0.0.7.tgz", @@ -751,6 +782,11 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -876,6 +912,11 @@ "bsert": "~0.0.10" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -944,12 +985,22 @@ "is-extglob": "^2.1.1" } }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -1126,6 +1177,11 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "regexpp": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", @@ -1157,6 +1213,11 @@ "glob": "^7.1.3" } }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, "semver": { "version": "7.3.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz", @@ -1323,6 +1384,27 @@ "bsert": "~0.0.10" } }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, "v8-compile-cache": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz", @@ -1350,6 +1432,20 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/package.json b/package.json index 11f6f91d8b..84046d2243 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "node": ">=8.0.0" }, "dependencies": { + "aws-sdk": "^2.829.0", "bcfg": "~0.1.6", "bcrypto": "~5.3.0", "bdb": "~1.3.0", @@ -46,8 +47,8 @@ "bweb": "~0.1.10", "goosig": "~0.9.0", "hs-client": "~0.0.9", - "n64": "~0.2.10", "mrmr": "~0.1.8", + "n64": "~0.2.10", "namebase-hs-client": "github:namebasehq/hs-client#18d9b0b", "urkel": "~0.6.3" },