diff --git a/install-types-registry/package.json b/install-types-registry/package.json deleted file mode 100644 index 208293eb..00000000 --- a/install-types-registry/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "types-registry": "latest" - } -} diff --git a/package-lock.json b/package-lock.json index 73c1a2b3..1b73098d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "integrity": "sha512-F9OalGhk60p/DnACfa1SWtmVTMni0+w9t/qfb5Bu7CsurkEjZFN7Z+ii/VGmYpaViPz7o3tBahRQae9O7skFlQ==", "dev": true, "requires": { - "@types/node": "8.0.31" + "@types/node": "8.0.32" } }, "@types/fs-extra": { @@ -19,7 +19,7 @@ "integrity": "sha512-PlKJw6ujJXLJjbvB3T0UCbY3jibKM6/Ya5cc9j1q+mYDeK3aR4Dp+20ZwxSuvJr9mIoPxp7+IL4aMOEvsscRTA==", "dev": true, "requires": { - "@types/node": "8.0.31" + "@types/node": "8.0.32" } }, "@types/mz": { @@ -28,13 +28,13 @@ "integrity": "sha1-pNgMCC/v5x5Ap8DwfR5lVbu8e1I=", "dev": true, "requires": { - "@types/node": "8.0.31" + "@types/node": "8.0.32" } }, "@types/node": { - "version": "8.0.31", - "resolved": "https://registry.npmjs.org/@types/node/-/node-8.0.31.tgz", - "integrity": "sha512-R+LdMJHJQwRd/Ca0Nr5KnwbSWHxTD3DWz4ivqoPeNH+YPcuirMWK+Ti9Mx32jOecmPhHOCd+6CefU5e1eVq2Ew==", + "version": "8.0.32", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.0.32.tgz", + "integrity": "sha512-n1zzgeQehndikZc/8N4rGSZc99cO6Tb3OInKzvWYniJsT/jet3m57buaBFa5cMeVNFosN4PKZ2LM1y16CFD7Rg==", "dev": true }, "@types/node-fetch": { @@ -43,7 +43,7 @@ "integrity": "sha1-UhB46PDGmhWOUCIAWsqS0mIPbVc=", "dev": true, "requires": { - "@types/node": "8.0.31" + "@types/node": "8.0.32" } }, "@types/oboe": { @@ -52,7 +52,7 @@ "integrity": "sha1-OvywzJSfzfd3hh2W6FcfO0hvD6c=", "dev": true, "requires": { - "@types/node": "8.0.31" + "@types/node": "8.0.32" } }, "@types/parsimmon": { @@ -72,17 +72,14 @@ "integrity": "sha512-9oVAi1Jlr274pbMGPEe0S3IPImV9knVNafa6E4MookD/fjOZAE6EmLkFX5ZjtZ9OXNPi2FCIZzUSMvwAUUKeSg==", "dev": true, "requires": { - "@types/node": "8.0.31" + "@types/node": "8.0.32" } }, - "@types/tar": { - "version": "1.0.29", - "resolved": "https://registry.npmjs.org/@types/tar/-/tar-1.0.29.tgz", - "integrity": "sha1-qvBknYO6pE9Ji/ezVP0ECtjIubM=", - "dev": true, - "requires": { - "@types/node": "8.0.31" - } + "@types/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha1-EHPEvIJHVK49EM+riKsCN7qWTk0=", + "dev": true }, "@types/yargs": { "version": "8.0.2", @@ -99,7 +96,7 @@ "date-utils": "1.2.21", "jws": "3.1.4", "node-uuid": "1.4.7", - "request": "2.82.0", + "request": "2.83.0", "underscore": "1.8.3", "xmldom": "0.1.27", "xpath.js": "1.0.7" @@ -405,14 +402,6 @@ "readable-stream": "2.0.6" } }, - "block-stream": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", - "integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=", - "requires": { - "inherits": "2.0.3" - } - }, "boom": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", @@ -480,6 +469,11 @@ "inherits": "2.0.3" } }, + "chownr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.0.1.tgz", + "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE=" + }, "cliui": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", @@ -643,13 +637,13 @@ "fs-promise": "2.0.3", "strip-json-comments": "2.0.1", "tslint": "5.7.0", - "typescript": "2.6.0-dev.20170926" + "typescript": "2.6.0-dev.20171005" }, "dependencies": { "typescript": { - "version": "2.6.0-dev.20170926", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-2.6.0-dev.20170926.tgz", - "integrity": "sha512-Hb8TZnA2fH75G8l7Al8gbv6KR/jXnvXXQbrq3k7gAHHAGo8wsLrPSTYqBdInfm9LmZKS+qi+Ix0vfTHxmpidow==" + "version": "2.6.0-dev.20171005", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-2.6.0-dev.20171005.tgz", + "integrity": "sha512-tnlYauRIL6QH3EYSmwEE+Rr/+a3nQruUJFbVCyIQL7r14hkpzLRz4uqV/Vix1Cmnf7Q1K4yU3l+YcoekVjM1tQ==" } } }, @@ -684,6 +678,14 @@ "iconv-lite": "0.4.19" } }, + "end-of-stream": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.0.tgz", + "integrity": "sha1-epDYM+/abPpurA9JSduw+tOmMgY=", + "requires": { + "once": "1.4.0" + } + }, "error-ex": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz", @@ -801,17 +803,6 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, - "fstream": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.11.tgz", - "integrity": "sha1-XB+x8RdHcRTwYyoOtLcbPLD9MXE=", - "requires": { - "graceful-fs": "4.1.11", - "inherits": "2.0.3", - "mkdirp": "0.5.1", - "rimraf": "2.6.2" - } - }, "gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", @@ -1460,9 +1451,9 @@ } }, "npm-registry-client": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/npm-registry-client/-/npm-registry-client-8.4.0.tgz", - "integrity": "sha512-PVNfqq0lyRdFnE//nDmn3CC9uqTsr8Bya9KPLIevlXMfkP0m4RpCVyFFk0W1Gfx436kKwyhLA6J+lV+rgR81gQ==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/npm-registry-client/-/npm-registry-client-8.5.0.tgz", + "integrity": "sha512-Nkcw24bfECKFNt0FLDQ+PjVqSlKxMggcboXiUBIvjbCnA15xjRO4kCwRDluGNXZjHFLx/vPjN4+ESXyVjpXLbQ==", "requires": { "concat-stream": "1.6.0", "graceful-fs": "4.1.11", @@ -1470,7 +1461,7 @@ "npm-package-arg": "5.1.2", "npmlog": "4.1.2", "once": "1.4.0", - "request": "2.82.0", + "request": "2.83.0", "retry": "0.10.1", "semver": "5.4.1", "slide": "1.1.6", @@ -1649,6 +1640,15 @@ "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, + "pump": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.2.tgz", + "integrity": "sha1-Oz7mUS+U8OV1U4wXmV+fFpkKXVE=", + "requires": { + "end-of-stream": "1.4.0", + "once": "1.4.0" + } + }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -1692,9 +1692,9 @@ } }, "request": { - "version": "2.82.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.82.0.tgz", - "integrity": "sha512-/QWqfmyTfQ4OYs6EhB1h2wQsX9ZxbuNePCvCm0Mdz/mxw73mjdg0D4QdIl0TQBFs35CZmMXLjk0iCGK395CUDg==", + "version": "2.83.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz", + "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", "requires": { "aws-sign2": "0.7.0", "aws4": "1.6.0", @@ -1750,14 +1750,6 @@ "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q=" }, - "rimraf": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", - "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", - "requires": { - "glob": "7.1.2" - } - }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -1863,11 +1855,6 @@ "safe-buffer": "5.1.1" } }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -1878,6 +1865,11 @@ "strip-ansi": "3.0.1" } }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", @@ -1911,14 +1903,26 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, - "tar": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", - "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", + "tar-fs": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.0.tgz", + "integrity": "sha512-I9rb6v7mjWLtOfCau9eH5L7sLJyU2BnxtEZRQ5Mt+eRKmf1F0ohXmT/Jc3fr52kDvjJ/HV5MH3soQfPL5bQ0Yg==", "requires": { - "block-stream": "0.0.9", - "fstream": "1.0.11", - "inherits": "2.0.3" + "chownr": "1.0.1", + "mkdirp": "0.5.1", + "pump": "1.0.2", + "tar-stream": "1.5.4" + } + }, + "tar-stream": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.4.tgz", + "integrity": "sha1-NlSc8E7RrumyowwBQyUiONr5QBY=", + "requires": { + "bl": "1.1.2", + "end-of-stream": "1.4.0", + "readable-stream": "2.0.6", + "xtend": "4.0.1" } }, "thenify": { @@ -1942,6 +1946,14 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "requires": { + "os-tmpdir": "1.0.2" + } + }, "tough-cookie": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.3.tgz", @@ -1969,13 +1981,13 @@ "resolve": "1.4.0", "semver": "5.4.1", "tslib": "1.7.1", - "tsutils": "2.8.2" + "tsutils": "2.11.2" } }, "tsutils": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.8.2.tgz", - "integrity": "sha1-LBSGukMSYIRbCsb5Aq/Z1wio6mo=", + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.11.2.tgz", + "integrity": "sha1-YBNgHjb6FP+VhBPlQdQn+4xqw0E=", "requires": { "tslib": "1.7.1" } @@ -2005,9 +2017,9 @@ "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, "typescript": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-2.5.2.tgz", - "integrity": "sha1-A4qV99m7tCCxvzW6MdTFwd0//jQ=" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-2.5.3.tgz", + "integrity": "sha512-ptLSQs2S4QuS6/OD1eAKG+S5G8QQtrU5RT32JULdZQtM1L3WTi34Wsu48Yndzi8xsObRAB9RPt/KhA9wlpEF6w==" }, "underscore": { "version": "1.8.3", diff --git a/package.json b/package.json index ede21dec..9469a83a 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,6 @@ "definitelytyped-header-parser": "Microsoft/definitelytyped-header-parser#production", "dtslint": "Microsoft/dtslint#production", "fs-extra": "4.0.0", - "fstream": "^1.0.10", "longjohn": "^0.2.11", "moment": "^2.18.1", "node-fetch": "^1.6.3", @@ -19,7 +18,8 @@ "oboe": "^2.1.3", "semver": "^5.3.0", "source-map-support": "^0.4.0", - "tar": "^2.2.1", + "tar-fs": "^1.16.0", + "tmp": "0.0.33", "typescript": "^2.4.0", "yargs": "^8.0.2" }, @@ -32,7 +32,7 @@ "@types/oboe": "^2.0.28", "@types/semver": "^5.3.30", "@types/source-map-support": "^0.4.0", - "@types/tar": "^1.0.27", + "@types/tmp": "0.0.33", "@types/yargs": "^8.0.1", "tslint": "^5.5.0" }, diff --git a/src/lib/azure-container.ts b/src/lib/azure-container.ts index ba376880..32be87a2 100644 --- a/src/lib/azure-container.ts +++ b/src/lib/azure-container.ts @@ -1,8 +1,7 @@ import { BlobService, common, createBlobService, ErrorOrResponse, ErrorOrResult } from "azure-storage"; import * as fs from "fs"; -import * as https from "https"; -import { streamDone, streamOfString, stringOfStream } from "../util/io"; +import { fetchResponse, streamDone, streamOfString, stringOfStream } from "../util/io"; import { gzip, unGzip } from "../util/tgz"; import { parseJson } from "../util/util"; @@ -83,23 +82,12 @@ export default class BlobWriter { } export async function readBlob(blobName: string): Promise { - return new Promise((resolve, reject) => { - const url = urlOfBlob(blobName); - const req = https.get(url as any, res => { - switch (res.statusCode) { - case 200: - if (res.headers["content-encoding"] !== "GZIP") { - reject(new Error(`${url} is not gzipped`)); - } else { - resolve(stringOfStream(unGzip(res))); - } - break; - default: - reject(new Error(`Can't get ${url}: ${res.statusCode} error`)); - } - }); - req.on("error", reject); - }); + const url = urlOfBlob(blobName); + const response = await fetchResponse(url); + if (response.headers["content-encoding"] !== "GZIP") { + throw new Error(`${url} is not gzipped`); + } + return stringOfStream(unGzip(response)); } export async function readJsonBlob(blobName: string): Promise { diff --git a/src/lib/npm-client.ts b/src/lib/npm-client.ts index 6c6135ac..078c620b 100644 --- a/src/lib/npm-client.ts +++ b/src/lib/npm-client.ts @@ -26,28 +26,28 @@ export default class NpmClient { async publish(publishedDirectory: string, packageJson: {}, dry: boolean): Promise { const readme = await readFile(joinPaths(publishedDirectory, "README.md")); - return new Promise((resolve, reject) => { - const body = createTgz(publishedDirectory, reject); - const metadata = { readme, ...packageJson }; + const body = createTgz(publishedDirectory); + const metadata = { readme, ...packageJson }; - const params = { - access: "public" as "public", - auth: this.auth, - metadata, - body - }; + const params = { + access: "public" as "public", + auth: this.auth, + metadata, + body + }; - if (dry) { - resolve(); - } else { - this.client.publish(npmRegistry, params, err => { - if (err) { - reject(err); - } else { - resolve(); - } - }); - } + if (dry) { + return; + } + + return new Promise((resolve, reject) => { + this.client.publish(npmRegistry, params, err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); }); } diff --git a/src/lib/versions.ts b/src/lib/versions.ts index 26d96766..4c48660d 100644 --- a/src/lib/versions.ts +++ b/src/lib/versions.ts @@ -161,8 +161,8 @@ async function fetchTypesPackageVersionInfo(pkg: AnyPackage, isPrerelease: boole } /** For use by publish-registry only. */ -export async function fetchLastPatchNumber(packageName: string): Promise { - return (await fetchVersionInfoFromNpm(packageName, /*isPrerelease*/ false))!.version.patch; +export async function fetchLastPatchNumber(escapedPackageName: string): Promise { + return (await fetchVersionInfoFromNpm(escapedPackageName, /*isPrerelease*/ false))!.version.patch; } async function fetchVersionInfoFromNpm( diff --git a/src/publish-registry.ts b/src/publish-registry.ts index d8f81362..3f060286 100644 --- a/src/publish-registry.ts +++ b/src/publish-registry.ts @@ -1,4 +1,3 @@ -import { remove } from "fs-extra"; import * as yargs from "yargs"; import { Options } from "./lib/common"; @@ -7,18 +6,18 @@ import { clearOutputPath } from "./lib/package-generator"; import { AllPackages, TypingsData } from "./lib/packages"; import { outputPath } from "./lib/settings"; import { fetchLastPatchNumber } from "./lib/versions"; -import { readJson, writeJson } from "./util/io"; +import { fetchResponse, writeJson } from "./util/io"; import { Logger, logger, writeLog } from "./util/logging"; -import { done, execAndThrowErrors, joinPaths, setDifference } from "./util/util"; +import { unGzipFileFromTar } from "./util/tgz"; +import { done, joinPaths, setDifference } from "./util/util"; const packageName = "types-registry"; +const indexJson = "index.json"; const registryOutputPath = joinPaths(outputPath, packageName); const readme = `This package contains a listing of all packages published to the @types scope on NPM. Generated by [types-publisher](https://github.com/Microsoft/types-publisher).`; -const registryInstallDir = joinPaths(__dirname, "..", "install-types-registry"); - if (!module.parent) { const dry = !!yargs.argv.dry; done(main(Options.defaults, dry)); @@ -28,9 +27,10 @@ export default async function main(options: Options, dry: boolean): Promise { - const oldRegistry = await downloadOldRegistry(); +async function shouldPublishNewRegistry(oldVersion: string, options: Options, log: Logger): Promise { + const oldRegistry = await downloadOldRegistry(oldVersion); const packages = await AllPackages.read(options); const oldPackagesList = Object.keys(oldRegistry.entries); const newPackagesList = packages.allTypings().map(t => t.name); @@ -61,25 +61,17 @@ async function shouldPublishNewRegistry(options: Options, log: Logger): Promise< return anyDiff; } -async function downloadOldRegistry(): Promise { - await clean(); // Just to be sure nothing was left behind - await execAndThrowErrors("npm install", registryInstallDir); - const res = readJson(joinPaths(registryInstallDir, "node_modules", "types-registry", "index.json")); - await clean(); - return res; - - async function clean(): Promise { - await remove(joinPaths(registryInstallDir, "node_modules")); - await remove(joinPaths(registryInstallDir, "package-lock.json")); - } +async function downloadOldRegistry(oldVersion: string): Promise { + const url = `https://registry.npmjs.org/${packageName}/-/${packageName}-${oldVersion}.tgz`; + const response = await fetchResponse(url); + return JSON.parse(await unGzipFileFromTar(response, `${packageName}/${indexJson}`)); } -async function generateAndPublishRegistry(log: Logger, dry: boolean): Promise { +async function generateAndPublishRegistry(log: Logger, newVersion: string, dry: boolean): Promise { // Don't include not-needed packages in the registry. const typings = await AllPackages.readTypings(); - const last = await fetchLastPatchNumber(packageName); - const packageJson = generatePackageJson(last + 1); + const packageJson = generatePackageJson(newVersion); await generate(typings, packageJson, log); await publish(packageJson, dry); @@ -88,7 +80,7 @@ async function generateAndPublishRegistry(log: Logger, dry: boolean): Promise, packageJson: {}, log: Logger): Promise { await clearOutputPath(registryOutputPath, log); await writeOutputFile("package.json", packageJson); - await writeOutputFile("index.json", generateRegistry(typings)); + await writeOutputFile(indexJson, generateRegistry(typings)); await writeOutputFile("README.md", readme); function writeOutputFile(filename: string, content: {}): Promise { @@ -101,10 +93,14 @@ async function publish(packageJson: {}, dry: boolean): Promise { await client.publish(registryOutputPath, packageJson, dry); } -function generatePackageJson(patch: number): {} { +function getVersion(patch: number): string { + return `0.1.${patch}`; +} + +function generatePackageJson(version: string): {} { return { name: packageName, - version: `0.1.${patch}`, + version, description: "A registry of TypeScript declaration file packages published within the @types scope.", repository: { type: "git", diff --git a/src/types/fstream.d.ts b/src/types/fstream.d.ts deleted file mode 100644 index a12c4c01..00000000 --- a/src/types/fstream.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function Reader(options: ReaderOptions): NodeJS.ReadableStream; -interface ReaderOptions { - path: string; - type: "Directory"; - filter(entry: FStreamEntry): boolean; -} -interface FStreamEntry { - props: { type: string, mode: number }; -} diff --git a/src/types/tar-fs.d.ts b/src/types/tar-fs.d.ts new file mode 100644 index 00000000..1c979e27 --- /dev/null +++ b/src/types/tar-fs.d.ts @@ -0,0 +1,5 @@ +import { Readable, Writable } from "stream"; + +export function pack(directoryName: string): Readable; + +export function extract(directoryName: string): Writable; diff --git a/src/util/io.ts b/src/util/io.ts index 1f060a19..8b8630c4 100644 --- a/src/util/io.ts +++ b/src/util/io.ts @@ -1,4 +1,5 @@ import { readFile as readFileWithEncoding, stat, writeFile as writeFileWithEncoding, writeJson as writeJsonRaw } from "fs-extra"; +import { get as httpsGet, IncomingMessage } from "https"; import fetch, { RequestInit, Response } from "node-fetch"; import * as stream from "stream"; @@ -18,6 +19,20 @@ export async function fetchJson(url: string, init?: RequestInit & { retries?: nu return parseJson(await response.text()); } +export async function fetchResponse(url: string): Promise { + return new Promise((resolve, reject) => { + const req = httpsGet(url, res => { + switch (res.statusCode) { + case 200: + resolve(res); + default: + reject(new Error(`Can't get ${url}: ${res.statusCode} error`)); + } + }); + req.on("error", reject); + }); +} + export function writeFile(path: string, content: string): Promise { return writeFileWithEncoding(path, content, { encoding: "utf8" }); } diff --git a/src/util/tgz.ts b/src/util/tgz.ts index a576ae60..8e241863 100644 --- a/src/util/tgz.ts +++ b/src/util/tgz.ts @@ -1,57 +1,32 @@ import { createWriteStream } from "fs"; -import { FStreamEntry, Reader } from "fstream"; -import { Pack } from "tar"; -import * as zlib from "zlib"; -import { streamDone } from "./io"; +import { extract, pack } from "tar-fs"; +import { dirSync as tmpDir } from "tmp"; +import { createGunzip, createGzip } from "zlib"; +import { readFile, streamDone } from "./io"; +import { joinPaths } from "./util"; export function gzip(input: NodeJS.ReadableStream): NodeJS.ReadableStream { - return input.pipe(zlib.createGzip()); + return input.pipe(createGzip()); } export function unGzip(input: NodeJS.ReadableStream): NodeJS.ReadableStream { - const output = zlib.createGunzip(); + const output = createGunzip(); input.pipe(output); return output; } +export async function unGzipFileFromTar(input: NodeJS.ReadableStream, file: string): Promise { + const cwd = tmpDir().name; + await streamDone(unGzip(input).pipe(extract(cwd))); + return readFile(joinPaths(cwd, file)); +} + export function writeTgz(inputDirectory: string, outFileName: string): Promise { - return new Promise((resolve, reject) => { - resolve(streamDone(createTgz(inputDirectory, reject).pipe(createWriteStream(outFileName)))); - }); + return streamDone(createTgz(inputDirectory).pipe(createWriteStream(outFileName))); } // To output this for testing: Export it and: // `require("./bin/lib/npm-client").createTgz("./output/foo", err => { throw err }).pipe(fs.createWriteStream("foo.tgz"))` -export function createTgz(dir: string, onError: (error: Error) => void): NodeJS.ReadableStream { - return gzip(createTar(dir, onError)); -} - -function createTar(dir: string, onError: (error: Error) => void): NodeJS.ReadableStream { - const packer = Pack({ noProprietary: true }) - .on("error", onError); - - return Reader({ path: dir, type: "Directory", filter: addDirectoryExecutablePermission }) - .on("error", onError) - .pipe(packer); -} - -/** - * Work around a bug where directories bundled on Windows do not have executable permission when extracted on Linux. - * https://github.com/npm/node-tar/issues/7#issuecomment-17572926 - */ -function addDirectoryExecutablePermission(entry: FStreamEntry): boolean { - if (entry.props.type === "Directory") { - entry.props.mode = addExecutePermissionsFromReadPermissions(entry.props.mode); - } - return true; -} - -function addExecutePermissionsFromReadPermissions(mode: number): number { - // Constant that gives execute permissions to owner, group, and others. "+x" - const allExecutePermissions = 0o111; - // Moves the bits for read permissions into the place for execute permissions. - // In other words, a component will have execute permissions if it has read permissions. - const readPermissionsAsExecutePermissions = (mode >>> 2) & allExecutePermissions; - // Add these additional execute permissions to the mode. - return mode | readPermissionsAsExecutePermissions; +export function createTgz(dir: string): NodeJS.ReadableStream { + return gzip(pack(dir)); }