diff --git a/package.json b/package.json index f07531d..9f58ef5 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "mkdirp": "~0.3.5", "my-local-ip": "~1.0.0", "nomnom": "~1.6.2", - "request": "~2.47.0", + "request": "^2.47.0", "semver": "^2.3.0", "structured-clone": "~0.2.2", "tar": "~0.1.18", diff --git a/src/bundle.js b/src/bundle.js index 21e7c30..bedc352 100644 --- a/src/bundle.js +++ b/src/bundle.js @@ -10,6 +10,7 @@ var net = require('net'); var fs = require('fs'); var path = require('path') + , https = require('https') , temp = require('temp') , colonyCompiler = require('colony-compiler') , async = require('async') @@ -18,6 +19,8 @@ var path = require('path') , effess = require('effess') , debug = require('debug')('tessel') , logs = require('../src/logs') + , request = require('request') + , zlib = require('zlib') ; // We want to force node-tar to not use extended headers. @@ -138,48 +141,170 @@ exports.bundleFiles = function (startpath, args, files, opts, next) }); }; +var MIPS_CACHE_PATH = "mips_modules"; +var MIPS_BINARY_ROOT = "https://s3.amazonaws.com/tessel-mips-modules/"; + +var fetchMipsModule = function (location, callback) +{ + // Read in the module version. + var filename = location + '/package.json' + var json = JSON.parse(fs.readFileSync(filename, 'utf8')); + var version = json.version; + + var name = location.match(/[^\/]+$/)[0]; + var mipsModulePath = path.join(MIPS_CACHE_PATH, name, version); + + // Cache the mips binaries from server. The cache is located at MIPS_CACHE_PATH + if (fs.existsSync(mipsModulePath)) { + logs.info("Found cached binary module", name, version) + return setTimeout(function () { callback(mipsModulePath) }); + } + + var packageUrl = MIPS_BINARY_ROOT + name + '_' + version + '.tgz'; + logs.info("Fetching binary module", packageUrl) + request({ + url: packageUrl, + gzip: true + }).pipe(zlib.createGunzip()).pipe(tar.Extract({ + path: mipsModulePath + })).on('end', function () { + logs.info("Fetched binary module", name, version) + callback(mipsModulePath); + }).on('error', function (err) { + fs.rmdirSync(mipsModulePath); + logs.err("No mips module found for", name, version, err); + process.exit(1); + }); +}; + +var isBinaryReleasePath = function (location) { + var match = location.match(/(node_modules\/.+)\/build\/Release$/); + return match && match[1]; +}; + +var createDirReader = function (root, base, filter) +{ + var reader = fstream.Reader({ + path: root, + type: "Directory", + filter: filter + }); + reader.basename = ""; + + var rebase = function (entry) { + if (base) { + entry.path = entry.path.replace(root, path.join(path.dirname(root), base)); + entry.on("entry", rebase); + } + else { + // entry.root.path = entry.path; + entry.root = {path: entry.path}; + } + }; + + reader.on('entry', rebase); + + reader.on('error', function (err) { + logs.err('Error bundling code archive: ' + err); + process.exit(1); + }); + + return reader; +}; // TODO should not be public, // relied on by debug push code path -exports.tarCode = function (dirpath, pushdir, next) +exports.tarCode = function (dirpath, options, next) { - var fstr = fstream.Reader({path: dirpath, type: "Directory"}) - fstr.basename = ''; + options = Object(options); - fstr.on('entry', function (e) { - e.root = {path: e.path}; - }) + var streams = []; + var packedData = []; - fstr.on('error', function (err) { - logs.err('Error bundling code archive: ' + err); + var packer = tar.Pack(); + packer._noProprietary = true; + + packer.on('error', function (err) { + logs.err('Error in compressing code archive: ' + err); process.exit(1); - }) + }); - var bufs = []; - var p = tar.Pack(); - p._noProprietary = true; - fstr.pipe(p).on('data', function (buf) { - bufs.push(buf); - }).on('end', function () { - var bundle = Buffer.concat(bufs); + var finalizePackage = function () { + var bundle = Buffer.concat(packedData); var hasIndex = false; - var p = tar.Parse().on('entry', function (a) { - if (a.path == '_start.js') { + var parser = tar.Parse().on('entry', function (entry) { + if (entry.path === '_start.js') { hasIndex = true; } }).on('end', function () { if (!hasIndex) { logs.err('Command line generated bundle without an /_start.js file. Please report this error.'); - process.exit(1); + // process.exit(1); } next(null, bundle); - }) - p.write(bundle); - p.end(); - }).on('error', function (err) { - logs.err('Error in compressing code archive: ' + err); - process.exit(1); - }); -} + }); + + parser.write(bundle); + parser.end(); + }; + + var startStream = function () { + var root = streams[0][0]; + var base = streams[0][1]; + + var stream = createDirReader(root, base, filter); + stream.on('end', onStreamEnd); + + var piped = stream.pipe(packer, {end: false}).on('data', function (data) { + packedData.push(data); + }); + + // Replace the arguments with the real stream. + streams.shift(); + streams.unshift(piped); + }; + + var onStreamEnd = function () { + streams.shift().removeAllListeners('data'); + + if (streams.length) { + startStream(); + } + else if (!pendingStreams) { + finalizePackage(); + } + }; + + var pendingStreams = 0; + var queueStream = function (root, base) { + streams.push([root, base]); + if (pendingStreams) { + pendingStreams -= 1; + if (Array.isArray(streams[0])) startStream(); + } + }; + + var filter = function (entry) { + if (entry.path.match(new RegExp(MIPS_CACHE_PATH + '$'))) return false; + if (entry.path.match(new RegExp(".git$"))) return false; + + // If this is a binary release, spawn a new reader for the corresponding + // mips binaries and filter our the x86 binaries. + var modulePath = isBinaryReleasePath(entry.path); + if (modulePath) { + fetchMipsModule(modulePath, function (mipsPath) { + queueStream(mipsPath, modulePath); + }); + + pendingStreams += 1; + return false; + } + + return true; + }; + + queueStream(dirpath, null); + startStream(); +}; diff --git a/src/index.js b/src/index.js index 67f5ab8..1e76f79 100644 --- a/src/index.js +++ b/src/index.js @@ -14,6 +14,7 @@ exports.Tessel = tessel_usb.Tessel; exports.findTessel = tessel_usb.findTessel; exports.listDevices = tessel_usb.listDevices; exports.bundleFiles = bundle.bundleFiles; +exports.tarCode = bundle.tarCode; require('./commands'); require('./script');