From b1da932ef2b77e28d85ca9cf75070fda8bf700d3 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 11 Oct 2018 06:18:00 -0700 Subject: [PATCH 01/43] Adding more granular node_modules creation --- internal/common/BUILD.bazel | 5 + internal/common/actions/BUILD.bazel | 3 + internal/common/actions/actions.bzl | 2 + .../actions/create_source_dir/BUILD.bazel | 6 + .../actions/create_source_dir/action.bzl | 50 ++ .../create_source_dir/create_source_dir.js | 246 ++++++++ internal/common/actions/run_js/BUILD.bazel | 6 + internal/common/actions/run_js/BazelAction.js | 126 ++++ internal/common/actions/run_js/README.md | 3 + internal/common/actions/run_js/action.bzl | 43 ++ internal/common/context.bzl | 285 +++++++++ internal/package.json | 1 + internal/ts_library/BUILD.bazel | 1 + internal/ts_library/compile.js | 67 +-- internal/ts_library/rule.bzl | 543 +++++++++--------- internal/ts_library/tsconfig.gen.js | 53 ++ internal/yarn.lock | 4 + 17 files changed, 1145 insertions(+), 299 deletions(-) create mode 100644 internal/common/BUILD.bazel create mode 100644 internal/common/actions/BUILD.bazel create mode 100644 internal/common/actions/actions.bzl create mode 100644 internal/common/actions/create_source_dir/BUILD.bazel create mode 100644 internal/common/actions/create_source_dir/action.bzl create mode 100644 internal/common/actions/create_source_dir/create_source_dir.js create mode 100644 internal/common/actions/run_js/BUILD.bazel create mode 100644 internal/common/actions/run_js/BazelAction.js create mode 100644 internal/common/actions/run_js/README.md create mode 100644 internal/common/actions/run_js/action.bzl create mode 100644 internal/common/context.bzl create mode 100644 internal/ts_library/tsconfig.gen.js diff --git a/internal/common/BUILD.bazel b/internal/common/BUILD.bazel new file mode 100644 index 0000000..15afdb9 --- /dev/null +++ b/internal/common/BUILD.bazel @@ -0,0 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "context.bzl" +]) diff --git a/internal/common/actions/BUILD.bazel b/internal/common/actions/BUILD.bazel new file mode 100644 index 0000000..ac8ec7b --- /dev/null +++ b/internal/common/actions/BUILD.bazel @@ -0,0 +1,3 @@ +package(default_visibility = ["//visibility:public"]) + +export_files(["actions.bzl"]) \ No newline at end of file diff --git a/internal/common/actions/actions.bzl b/internal/common/actions/actions.bzl new file mode 100644 index 0000000..9b1fc53 --- /dev/null +++ b/internal/common/actions/actions.bzl @@ -0,0 +1,2 @@ +load("//internal/common/actions/run_js:action.bzl", "run_js") +load("//internal/common/actions/create_source_dir:action.bzl", "create_source_dir") diff --git a/internal/common/actions/create_source_dir/BUILD.bazel b/internal/common/actions/create_source_dir/BUILD.bazel new file mode 100644 index 0000000..0fd2399 --- /dev/null +++ b/internal/common/actions/create_source_dir/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "action.bzl", + "create_source_dir.js", +]) \ No newline at end of file diff --git a/internal/common/actions/create_source_dir/action.bzl b/internal/common/actions/create_source_dir/action.bzl new file mode 100644 index 0000000..0a2a5cb --- /dev/null +++ b/internal/common/actions/create_source_dir/action.bzl @@ -0,0 +1,50 @@ +def create_source_dir(js, js_source, create_dir): + """Creates a directory with the sources described in the JsSource object + + Args: + js: JsContext object + js_source_provider: JsSource object describing the sources to symlink in the directory + source_dir: File object to populate with the sources (eg. ctx.outputs.compilation_src_dir) + + Returns: + Array with JsSource provider in it + """ + + library = js_source.library + gen_scripts = [] + if js_source.gen_scripts: + gen_scripts += js_source.gen_scripts + + direct_inputs = [] + direct_inputs += library.module_paths + transitive_inputs = [library.all_sources] + # Depset with all of the sources in it + + script_args = js.script_args(js, js._create_source_dir_js) + script_args.add("--into", create_dir) + script_args.add("--from", js.package_path) + print(library.all_sources) + print(library.module_paths) + print(library.all_modules) + + for gen_script in gen_scripts: + if type(gen_script) == type(""): + direct_inputs.append(gen_script) + script_args.add("g:./{}".format(gen_script.path)) + elif type(gen_script) == type([]) and len(gen_script) > 0: + argValue = "g" + for value in gen_script: + direct_inputs.append(value) + argValue += ":./{}".format(value.path) + script_args.add(argValue) + + script_args.add_all(library.all_sources, format_each = "s:%s") + script_args.add_all(library.module_paths, format_each = "mrs:%s/node_modules/") + script_args.add_all(library.all_modules , format_each = "m:%s") + + inputs = depset( + direct = direct_inputs, + transitive = transitive_inputs, + ) + + js.run_js(js, inputs = inputs, outputs = [create_dir], script = js._create_source_dir_js, script_args = script_args) \ No newline at end of file diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js new file mode 100644 index 0000000..f19e272 --- /dev/null +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -0,0 +1,246 @@ +const fs = require("fs-extra"); +const path = require("path"); +const { + BazelAction, + safeSymlink, + ensureArray +} = require("../run_js/BazelAction"); + +/** + * Action for creating a directory with the passed in files for symlinking and + * copying. + * + * The files can follow a uri syntax with params: + * recurse=false (default) + * symlink=true (default) + * + * + * The files to be populated have flags set on them for the appropriate actions: + * [flags]:./file/path + * Possible flags: + * s: Symlink + * s:./some/dir/ + * c: Copy + * c:/some/file + * r: Recurse + * rs:./some/dir/ + * m: Module (put it in node_modules) + * mr: Recursive module (add all the modules to node_modules) + * g: Generate (run the passed in script to generate source files + * + * eg. node create_source_dir.js -s file/to/symlink. + */ +BazelAction( + { + string: [ + // Root path to symlink/copy from + "from", + // The folder to populate + "into" + ] + }, + async args => { + const { current_target, workspace_name, package_path, from, into } = args; + const sources = ensureArray(args._); + const nodeModulesPath = path.join(into, "node_modules"); + const package = { + workspace: workspace_name, + path: package_path + }; + + makeDirectory(into); + const existingDirs = new Set([into]); + + const populateFiles = async source => { + const parsed = parseSource(source); + console.dir(parsed); + /** + * Node Module Population + */ + if (parsed.module) { + if (parsed.recurse) { + const fromNodeModulesDir = parsed.path; + const allModuleDirectories = fs.readdirSync(fromNodeModulesDir); + for (const moduleDirectory of allModuleDirectories) { + if (isOrgScopeDirectory(moduleDirectory)) { + const orgScopeFrom = path.join( + fromNodeModulesDir, + moduleDirectory + ); + const orgScopeInto = path.join(nodeModulesPath, moduleDirectory); + makeDirectory(orgScopeInto); + const allOrgModuleDirectories = fs.readdirSync(orgScopeFrom); + for (const orgModuleDirectory of allOrgModuleDirectories) { + makeSymlink( + path.join(orgScopeFrom, orgModuleDirectory), + path.join(orgScopeInto, orgModuleDirectory) + ); + } + } else { + makeSymlink( + path.join(fromNodeModulesDir, moduleDirectory), + path.join(nodeModulesPath, moduleDirectory) + ); + } + } + } + } else if (parsed.symlink) { + makeSymlink(parsed.path, path.join(into, parsed.path)); + } else if (parsed.generate) { + await makeGeneratedFiles(package, into, parsed.path, parsed.params); + } + }; + + console.log(`Current Target: ${current_target}`); + console.log(`Target Source Path: ${from}`); + console.log(`Copying into: ${into}`); + console.log(`Running in [${process.cwd()}]`); + console.log(`Environment is: ${JSON.stringify(process.env, null, 2)}`); + console.log(`Will create [${into}] for populating`); + + for (const source of sources) { + await populateFiles(source); + } + + // const internalDeps = joinedInternalDeps.split("|"); + // const srcs = joinedSrcs.split("|"); + + // fs.mkdirSync(into); + // safeSymlink( + // path.join(installedNpmPackagesDir, "node_modules"), + // path.join(destinationDir, "node_modules") + // ); + + // // Copy every internal dependency into the appropriate location. + // for (const internalDep of internalDeps) { + // if (!internalDep) { + // continue; + // } + // const [joinedSrcs, compiledDir] = internalDep.split(":"); + // const srcs = joinedSrcs.split(";"); + // for (const src of srcs) { + // if (!src) { + // continue; + // } + // safeSymlink( + // path.join(compiledDir, src), + // path.join(destinationDir, src) + // ); + // } + // } + + // // Extract compiler options from tsconfig.json, overriding anything other + // // than compiler options. + // const originalTsConfig = JSON.parse(fs.readFileSync(tsconfigPath, "utf8")); + + // // Copy source code and update import statements in this target's sources. + // for (const src of srcs) { + // if (!src) { + // continue; + // } + // if (!fs.existsSync(src)) { + // console.error(` + // Missing file ${src} required by ${targetLabel}. + // `); + // process.exit(1); + // } + // const destinationFilePath = path.join(destinationDir, src); + // fs.ensureDirSync(path.dirname(destinationFilePath)); + // safeSymlink(src, destinationFilePath); + // } + + // const compilerOptions = {}; + // Object.assign(compilerOptions, originalTsConfig.compilerOptions || {}); + // Object.assign(compilerOptions, { + // moduleResolution: "node", + // declaration: true, + // rootDir: "." + // }); + // fs.writeFileSync( + // path.join(destinationDir, "tsconfig.json"), + // JSON.stringify( + // { + // compilerOptions, + // files: srcs.filter(src => src.endsWith(".ts") || src.endsWith(".tsx")) + // }, + // null, + // 2 + // ), + // "utf8" + // ); + } +); + +function makeDirectory(directory) { + fs.ensureDirSync(directory); +} + +function makeSymlink(from, to) { + fs.ensureSymlinkSync(from, to); +} + +async function writeFile(into, output) { + if (!path) { + throw new Error(`Attempted to write file with no path set`); + } + await fs.writeFile(path.join(into, output.path), output.body); +} + +async function makeGeneratedFiles(package, into, generatorScript, inputFiles) { + const resolvedScript = path.join(process.cwd(), generatorScript); + const generator = require(resolvedScript); + if (!generator) { + throw new Error(`No generator script found at ${generatorScript}`); + } + const inputFileContents = await Promise.all( + inputFiles.map(async path => ({ + path, + body: await fs.readFile(path) + })) + ); + const outputFiles = await generator({ + package, + into, + inputs: inputFileContents + }); + await Promise.all(outputFiles.map(output => writeFile(into, output))); +} + +function isOrgScopeDirectory(directory) { + return directory.startsWith("@"); +} + +function parseSource(source) { + const [flags, sourcePath, ...params] = source.split(":"); + const actions = { path: sourcePath, params }; + let flagIndex = 0; + while (flagIndex < flags.length) { + const currentFlag = flags[flagIndex]; + switch (currentFlag) { + case "s": + actions["symlink"] = true; + break; + case "c": + actions["copy"] = true; + break; + case "r": + actions["recurse"] = true; + break; + case "m": + actions["module"] = true; + break; + case "g": + actions["generate"] = true; + break; + default: + throw new Error( + `Encountered invalid flag "${currentFlag}" in "${source}"` + ); + } + flagIndex++; + } + if (actions.symlink && actions.copy) { + throw new Error(`Attempted to both symlink and copy ${source}`); + } + return actions; +} diff --git a/internal/common/actions/run_js/BUILD.bazel b/internal/common/actions/run_js/BUILD.bazel new file mode 100644 index 0000000..17669fd --- /dev/null +++ b/internal/common/actions/run_js/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "action.bzl", + "BazelAction.js", +]) \ No newline at end of file diff --git a/internal/common/actions/run_js/BazelAction.js b/internal/common/actions/run_js/BazelAction.js new file mode 100644 index 0000000..2841687 --- /dev/null +++ b/internal/common/actions/run_js/BazelAction.js @@ -0,0 +1,126 @@ +const fs = require("fs-extra"); +const getopts = require("getopts"); +const path = require("path"); +const readline = require("readline"); + +/** + * Wrap a callback that will be called with the script arguments + * + * The arguments passed to the callback have only the arguments intended for + * the script and have removed any escaping created by the run_js_action bazel + * rule. + * + * The opts.args will be passed to the options parser. + * + * @param { { args: Object } } opts + * @param {(ctx: { args: Object }) => void | Promise} cb + */ +async function BazelAction(opts, cb) { + try { + const args = await parseBazelArgs(opts.args); + await cb(args); + } catch (e) { + console.error(e); + process.exit(1); + } +} + +/** + * Reads in param file produced by bazel + * + * The param file format should be "multiline" + * + * @param {string} filePath + */ +function readParamFile(filePath) { + return new Promise((resolve, reject) => { + try { + const rl = readline.createInterface({ + input: fs.createReadStream(filePath), + terminal: false, + crlfDelay: Infinity + }); + + argfileArgs = []; + + rl.on("line", line => argfileArgs.push(line)); + rl.on("close", () => resolve(argfileArgs)); + } catch (e) { + reject(e); + } + }); +} + +/** + * Wrap the passed in argument in an array or return the argument if it is + * already an array + * + * @param {any} arrayOrFirstElement + */ +function ensureArray(arrayOrFirstElement) { + if (Array.isArray(arrayOrFirstElement)) { + return arrayOrFirstElement; + } else { + return [arrayOrFirstElement]; + } +} + +/** + * Merge the two objects, if both objects have the same key then the values are + * concatenated into an array. + * @param {Object} into + * @param {Object} from + */ +function mergeArgs(into, from) { + for (const argName in from) { + if (into[argName]) { + into[argName] = ensureArray(into[argName]).concat(from[argName]); + } else { + into[argName] = from[argName]; + } + } + return into; +} + +/** + * Parse shell arguments with automatic handling for --params {paramFile} + * + * Uses mri (basically same options as minimist) to parse args. If there is a + * parmeter --params={paramFile} in the arguments, then the param file will + * automatically be read and the arguments in it returned in the + * + * @param {Object} opts Options for getopts (https://www.npmjs.com/package/getopts) + * @param {string[]} args Array of arguments to parse (defaults to process.argv.slice(2)) + * @returns {{[argName: string]: ArgValue}} Object with arg names as keys + */ +async function parseBazelArgs(opts) { + const result = getopts(process.argv.slice(2), opts); + + // --params was chosen as the use_param_file param_file_arg to match precident in rules_go + // https://github.com/bazelbuild/rules_go/blob/2179a6e1b576fc2a309c6cf677ad40e5b7f999ba/go/private/context.bzl#L87 + if (result.params) { + const argFileArgs = await readParamFile(result.params); + const argfileResult = getopts(argFileArgs, opts); + mergeArgs(result, argfileResult); + } + + return result; +} + +function safeSymlink(fromPath, toPath) { + const oldWorkingDir = process.cwd(); + const destinationPathDir = path.dirname(toPath); + fs.ensureDirSync(destinationPathDir); + process.chdir(destinationPathDir); + fs.symlinkSync( + path.relative(destinationPathDir, fromPath), + path.basename(toPath) + ); + process.chdir(oldWorkingDir); +} + +module.exports = { + safeSymlink, + ensureArray, + BazelAction, +} \ No newline at end of file diff --git a/internal/common/actions/run_js/README.md b/internal/common/actions/run_js/README.md new file mode 100644 index 0000000..ccbdb39 --- /dev/null +++ b/internal/common/actions/run_js/README.md @@ -0,0 +1,3 @@ +# run_js Action + +This should be invoked using the [js_context](../../context.bzl) \ No newline at end of file diff --git a/internal/common/actions/run_js/action.bzl b/internal/common/actions/run_js/action.bzl new file mode 100644 index 0000000..9f57897 --- /dev/null +++ b/internal/common/actions/run_js/action.bzl @@ -0,0 +1,43 @@ +load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") + +def run_js( + js, + inputs, + outputs, + script, + script_args): + """Create action that uses nodejs to run an internal .js file + + This is intended to be used internally by bazel-javascript and should + only have access to the node_modules that bazel-javascript installs internally + + Args: + js: JsContext object. + inputs: additional depset of action inputs (e.g. source files) + outputs: the outputs that the action generates as a sequence of Files + script: .js File to run + script_args: arguments to pass to the js file + """ + + action_inputs = depset( + direct = [ + script, + js._actions_bazel_action_js, + js._internal_packages[NpmPackagesInfo].installed_dir, + ], + transitive = [ + inputs, + ], + ) + + env = { + "NODE_PATH": js._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", + } + + js.actions.run( + inputs = action_inputs, + outputs = outputs, + executable = js._internal_nodejs, + arguments = [script.path, script_args], + env = env, + ) \ No newline at end of file diff --git a/internal/common/context.bzl b/internal/common/context.bzl new file mode 100644 index 0000000..77f4363 --- /dev/null +++ b/internal/common/context.bzl @@ -0,0 +1,285 @@ +load("//internal/common/actions:actions.bzl", "create_source_dir", "run_js") +load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") +# load("@rules_nodejs//internal/common:node_module_info.bzl", "NodeModuleInfo", "collect_node_modules_aspect") +# load("@rules_nodejs//internal/common:module_mappings.bzl", "get_module_mappings", "module_mappings_runtime_aspect") + +############################################################################### +# Providers + +# The Providers used are inspired by the rules_go providers: +# https://github.com/bazelbuild/rules_go/blob/master/go/providers.rst#GoLibrary + +# The NodeModuleInfo is provided by the rules_nodejs and generated by the +# collect_node_modules_aspect when the target has a "tags" attribute with +# "NODE_MODULE_MARKER" in it. +# NodeModuleInfo = provider( +# fields = { +# Only the workspace is needed because rules_ +# "workspace": "The workspace name that the npm dependencies are provided from" +# }) + +# Wrapper for rule ctx that should be created through js_context +JsContext = provider() + +JsLibraryInfo = provider( + fields = [ + # The label that produced the JsLibrary + "label", + # Path of the BUILD.bazel file relative to the workspace root. + "package_path", + # Source files provided as input. + "srcs", + # All source files provided as input + "all_sources", + # Entire paths to add to npm modules (eg. a node_modules path) + "module_paths", + # Depset of JsModule that this library depends on + "modules", + # All modules that this library depends on + "all_modules", + ], +) +"""Metadata about an individual js library. + +This provider keeps track of bazel information for Javascript targets, such +as: module dependencies, source files, paths to merge into node_modules. +""" + +JsSourceInfo = provider(fields = [ + # The source js files + "srcs", + # List of scripts to output + "gen_scripts", + # The library rule that generated this source + "library", +]) +"""Source that will be used either directly or for transpilation to javascript +""" + +JsModule = provider( + fields = [ + # The root of the workspace + "workspace_name", + # The root of the module + "module_root", + # Name that will be used for non-relative imports + "module_name", + # Transitive dependencies for this module + "all_modules", + ], +) +""" Wraps a JsLibrary with a package_name for nonrelative imports + +Including this as a dependency should add the "package_name" as a key for +nonrelative imports +""" + +JsModuleMap = provider( + fields = [ + "module_map" + ] +) + +############################################################################### +# Common Attributes + +# Attributes that should be included for any rule that wants to create a JsContext +JS_CONTEXT_ATTRIBUTES = { + "_actions_bazel_action_js": attr.label( + allow_files = True, + single_file = True, + default = Label("//internal/common/actions/run_js:BazelAction.js"), + ), + "_create_source_dir_js": attr.label( + allow_files = True, + single_file = True, + default = Label("//internal/common/actions/create_source_dir:create_source_dir.js"), + ), + "_internal_nodejs": attr.label( + allow_files = True, + single_file = True, + default = Label("@nodejs//:node"), + ), + "_internal_packages": attr.label( + default = Label("//internal:packages"), + ), + "_empty_npm_packages": attr.label( + allow_files = True, + single_file = True, + default = Label("//internal/npm_packages/empty:packages"), + ), +} + +# Attributes that should be included for any rule that wants to create a JsLibrary +JS_LIBRARY_ATTRIBUTES = dict(JS_CONTEXT_ATTRIBUTES, **{ + "srcs": attr.label_list( + allow_files = True, + mandatory = True, + ), + "deps": attr.label_list( + default = [], + ), + "module_name": attr.string(), +}) + +RULES_NODEJS_MODULE_ATTRIBUTES = { + # The official bazel rules use module_name and module_root for non-relative + # module mapping. If the module_root is supplied and the module_name is not + # present then the module_name is assumed to be the target name. See: + # https://github.com/bazelbuild/rules_nodejs/blob/master/internal/common/module_mappings.bzl + "module_name": attr.string(), + "module_root": attr.string(), + # The official bazel rules look for this tag + "tags": ["NODE_MODULE_MARKER"], +} + +############################################################################### +# Helpers + +def _js_library_info(js, attr = None): + """Create a JsLibrary provider with attr.srcs and the sources from attr.deps + + Args: + js: JsContext object + attr: rule attributes to extract srcs and deps from + """ + + if not attr: + attr = js.attr + + # The srcs should contain what has been explicitly added for a rule + srcs_attr = getattr(attr, "srcs", []) + src_files = js._ctx.files.srcs + + # The deps is list of labels that should have providers that we can get sources from + deps_attr = getattr(attr, "deps", []) + + transitive_sources = [] + direct_modules = [] + transitive_modules = [] + module_paths = [] + + # Iterate through the deps to add them to their correct JsLibrary attributes + for dep in deps_attr: + print(dep) + if JsLibraryInfo in dep: + dep_js_library = dep[JsLibraryInfo] + # The dependency is another JsLibrary + transitive_sources.append(dep_js_library.all_sources) + transitive_modules.append(dep_js_library.all_modules) + elif hasattr(dep, "tags") and "NODE_MODULES_MARKER" in getattr(dep, "tags"): + # The dependency is a module defined by rules_nodejs + direct_modules += dep + elif NpmPackagesInfo in dep: + # The dependency is a node_modules directory installed by npm_packages + module_paths.append(dep[NpmPackagesInfo].installed_dir) + + + all_src_files = depset( + direct = src_files, + transitive = transitive_sources, + ) + + all_modules = depset( + direct = direct_modules, + transitive = transitive_modules, + ) + + return JsLibraryInfo( + package_path = js.package_path, + srcs = src_files, + all_sources = all_src_files, + module_paths = module_paths, + modules = direct_modules, + all_modules = all_modules, + ) + +def _library_to_source_info(js, library, gen_scripts = None): + """Create a JsSource provider for a given library + The library is a target, but this resolves the actual source files needed + to build the js library. + """ + + return JsSourceInfo( + srcs = library.all_sources, + gen_scripts = gen_scripts, + library = library, + ) + + +def _script_args(js, script_file): + """Create Args object that can be used with js.run_js() + + Args: + js: JsContext object + script_file: File object for the script to be run + """ + if script_file == None: + fail("No script file provided to script args") + + args = js.actions.args() + + # If the args get too big then spill over into the param file + args.use_param_file("--param=%s") + args.set_param_file_format("multiline") + + # args.add(script_file) + + args.add("--current_target", js.label) + args.add("--workspace_name", js.workspace_name) + args.add("--package_path", js.package_path) + + return args + +def _module_mappings(js): + """Get the hash of {module_name - module_root} + + The underlying function from rules_nodejs goes through all the + dependencies looking for + """ + print("The nodejs rules don't export the module mappings") + # get_module_mappings(js.label, js.attr) + +# Following pattern similar to rules_go +# https://github.com/bazelbuild/rules_go/blob/2179a6e1b576fc2a309c6cf677ad40e5b7f999ba/go/private/context.bzl#L207 +def js_context(ctx, attr = None): + if not attr: + attr = ctx.attr + + # Node js to be used to run javascript backed bazel actions + _internal_nodejs = getattr(ctx.file, "_internal_nodejs") + + # Packages that will be made available to javascript backed bazel actions + _internal_packages = getattr(attr, "_internal_packages") + + # Packages that will be used if none are provided + _empty_npm_packages = getattr(attr, "_empty_npm_packages") + + _actions_bazel_action_js = getattr(ctx.file, "_actions_bazel_action_js") + _create_source_dir_js = getattr(ctx.file, "_create_source_dir_js") + + return JsContext( + # Base Context + label = ctx.label, + attr = ctx.attr, + + # Fields + workspace_name = ctx.workspace_name, + package_path = ctx.label.package, + _ctx = ctx, + _internal_nodejs = _internal_nodejs, + _internal_packages = _internal_packages, + _empty_npm_packages = _empty_npm_packages, + _actions_bazel_action_js = _actions_bazel_action_js, + _create_source_dir_js = _create_source_dir_js, + + #Actions + actions = ctx.actions, + create_source_dir = create_source_dir, + run_js = run_js, + + #Helpers + script_args = _script_args, + library_info = _js_library_info, + library_to_source_info = _library_to_source_info, + ) diff --git a/internal/package.json b/internal/package.json index 1e14be8..ed7ef23 100644 --- a/internal/package.json +++ b/internal/package.json @@ -10,6 +10,7 @@ "css-loader": "^1.0.0", "file-loader": "^1.1.11", "fs-extra": "^6.0.1", + "getopts": "^2.2.1", "html-webpack-plugin": "^3.2.0", "mini-css-extract-plugin": "^0.4.1", "optimize-css-assets-webpack-plugin": "^4.0.3", diff --git a/internal/ts_library/BUILD.bazel b/internal/ts_library/BUILD.bazel index ddbe8d3..34c8465 100644 --- a/internal/ts_library/BUILD.bazel +++ b/internal/ts_library/BUILD.bazel @@ -6,4 +6,5 @@ exports_files([ "default_tsconfig.json", "rule.bzl", "transpile.js", + "tsconfig.gen.js", ]) diff --git a/internal/ts_library/compile.js b/internal/ts_library/compile.js index db78c00..a7fc23f 100644 --- a/internal/ts_library/compile.js +++ b/internal/ts_library/compile.js @@ -1,40 +1,41 @@ +const { + BazelAction, + safeSymlink +} = require("../common/actions/run_js/BazelAction"); const child_process = require("child_process"); const fs = require("fs-extra"); const path = require("path"); -const { safeSymlink } = require("../common/symlink"); -const [nodePath, scriptPath, fullSrcDir, destinationDir] = process.argv; - -// Copy over any non-TypeScript files (e.g. CSS assets). -copyNonTypeScriptFiles("."); +BazelAction({}, async ({ project, outDir }) => { + const copyNonTypeScriptFiles = dirRelativePath => { + for (const fileName of fs.readdirSync( + path.join(project, dirRelativePath) + )) { + const relativeFilePath = path.join(dirRelativePath, fileName); + const srcFilePath = path.join(project, relativeFilePath); + let destFilePath = path.join(outDir, relativeFilePath); + fs.ensureDirSync(path.dirname(destFilePath)); + if (fs.lstatSync(srcFilePath).isDirectory()) { + copyNonTypeScriptFiles(relativeFilePath); + } else if ( + fileName !== "node_modules" && + !fileName.endsWith(".ts") && + !fileName.endsWith(".tsx") + ) { + // Symlink any file that isn't a TypeScript file (e.g. precompile JS or CSS assets). + safeSymlink(srcFilePath, destFilePath); + } + } + }; -// Compile with TypeScript. -child_process.execSync( - `${ - process.env.NODE_PATH - }/.bin/tsc --project ${fullSrcDir} --outDir ${destinationDir}`, - { - stdio: "inherit" - } -); + // Copy over any non-TypeScript files (e.g. CSS assets). + copyNonTypeScriptFiles("."); -function copyNonTypeScriptFiles(dirRelativePath) { - for (const fileName of fs.readdirSync( - path.join(fullSrcDir, dirRelativePath) - )) { - const relativeFilePath = path.join(dirRelativePath, fileName); - const srcFilePath = path.join(fullSrcDir, relativeFilePath); - let destFilePath = path.join(destinationDir, relativeFilePath); - fs.ensureDirSync(path.dirname(destFilePath)); - if (fs.lstatSync(srcFilePath).isDirectory()) { - copyNonTypeScriptFiles(relativeFilePath); - } else if ( - fileName !== "node_modules" && - !fileName.endsWith(".ts") && - !fileName.endsWith(".tsx") - ) { - // Symlink any file that isn't a TypeScript file (e.g. precompile JS or CSS assets). - safeSymlink(srcFilePath, destFilePath); + // Compile with TypeScript. + child_process.execSync( + `${process.env.NODE_PATH}/.bin/tsc --project ${project} --outDir ${outDir}`, + { + stdio: "inherit" } - } -} + ); +}); diff --git a/internal/ts_library/rule.bzl b/internal/ts_library/rule.bzl index 3c033b1..e354b4b 100644 --- a/internal/ts_library/rule.bzl +++ b/internal/ts_library/rule.bzl @@ -1,286 +1,297 @@ load("//internal/js_library:rule.bzl", "JsLibraryInfo") load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") +load("//internal/common:context.bzl", "JS_LIBRARY_ATTRIBUTES", "js_context") -TsLibraryInfo = provider(fields=[ - # Directory containing the TypeScript files (and potentially other assets). - "original_typescript_dir", - # Source files provided as input. - "typescript_source_files", - # Directory containing the generated TypeScript definitions and compiled JavaScript. - "compiled_typescript_dir", +TsLibraryInfo = provider(fields = [ + # Directory containing the TypeScript files (and potentially other assets). + "original_typescript_dir", + # Source files provided as input. + "typescript_source_files", + # Directory containing the generated TypeScript definitions and compiled JavaScript. + "compiled_typescript_dir", ]) def _ts_library_impl(ctx): - # Ensure that we depend on at most one npm_packages, since we don't want to - # have conflicting package versions coming from separate node_modules - # directories. - direct_npm_packages = [ - dep - for dep in ctx.attr.deps - if NpmPackagesInfo in dep - ] - if len(direct_npm_packages) > 1: - fail("Found more than one set of NPM packages in target definition: " + ",".join([ - dep.label - for dep in direct_npm_packages - ])) - extended_npm_packages = depset( - direct = direct_npm_packages, - transitive = [ - dep[JsLibraryInfo].npm_packages - for dep in ctx.attr.deps - if JsLibraryInfo in dep - ], - ) - npm_packages_list = extended_npm_packages.to_list() - if len(npm_packages_list) > 1: - fail("Found more than one set of NPM packages through dependencies: " + ",".join([ - dep.label - for dep in npm_packages_list - ])) - # If we depend on an npm_packages target, we'll use its node_modules - # directory to find modules. Otherwise, we'll use an empty node_modules - # directory. - npm_packages = ( - npm_packages_list[0] if len(npm_packages_list) == 1 - else ctx.attr._empty_npm_packages - ) - # Gather all internal deps (other ts_library rules). - internal_deps = depset( - direct = [ - dep - for dep in ctx.attr.deps - if JsLibraryInfo in dep - ], - transitive = [ - dep[JsLibraryInfo].internal_deps - for dep in ctx.attr.deps - if JsLibraryInfo in dep - ], - ) + js = js_context(ctx) + # for i, d in enumerate(ctx.attr.deps): + # print(" {}. label = {}".format(i + 1, d.label)) + # print(" files = " + str([f.path for f in d.files.to_list()])) - # Create two directories that contain: - # - source files (including all internal dependencies) - # - node_modules (symlinked to installed external dependencies directory) + js_library = js.library_info(js) + js_source = js.library_to_source_info(js, js_library, gen_scripts = [ + [ctx.file._ts_config_genscript, ctx.file.tsconfig] + ]) - # First version includes all dependencies' TypeScript definitions, which - # requires compiling everything up the tree (slow). Necessary to be able - # to compile TypeScript, including type verification. - _ts_library_create_full_src( - ctx, - internal_deps, - npm_packages, - ctx.outputs.compilation_src_dir, - True, - ) + js.create_source_dir(js, js_source, ctx.outputs.compilation_src_dir) - # Second version only includes dependencies' transpiled JavaScript code, - # which is a lot faster but does not do any type checking. - _ts_library_create_full_src( - ctx, - internal_deps, - npm_packages, - ctx.outputs.transpilation_src_dir, - False, - ) + compile_args = js.script_args(js, ctx.file._ts_library_compile_script.path) + compile_args.add("--project", ctx.outputs.compilation_src_dir) + compile_args.add("--outDir", ctx.outputs.compiled_dir) - # Compile the directory with `tsc` (slower but stricter). - _ts_library_compile( - ctx, - internal_deps, - npm_packages, - ) + ts_compile_inputs = depset( + direct = [ctx.outputs.compilation_src_dir], + transitive = [], + ) - # Transpile the directory with `tsc` (faster, no type checking). - _ts_library_transpile( - ctx, - internal_deps, - npm_packages, - ) + js.run_js( + js, + inputs = ts_compile_inputs, + outputs = [ctx.outputs.compiled_dir], + script = ctx.file._ts_library_compile_script, + script_args = compile_args, + ) - return [ - JsLibraryInfo( - build_file_path = ctx.build_file_path, - javascript_source_files = [_compiled_extension(f.path) for f in ctx.files.srcs], - compiled_javascript_dir = ctx.outputs.transpiled_dir, - internal_deps = internal_deps, - npm_packages = extended_npm_packages, - npm_packages_installed_dir = npm_packages[NpmPackagesInfo].installed_dir, - ), - TsLibraryInfo( - original_typescript_dir = ctx.outputs.compilation_src_dir, - compiled_typescript_dir = ctx.outputs.compiled_dir, - typescript_source_files = [f.path for f in ctx.files.srcs], - ), - ] + return [ + js_library, + js_source, + ] + + # Ensure that we depend on at most one npm_packages, since we don't want to + # have conflicting package versions coming from separate node_modules + # directories. + # direct_npm_packages = [ + # dep + # for dep in ctx.attr.deps + # if NpmPackagesInfo in dep + # ] + # if len(direct_npm_packages) > 1: + # fail("Found more than one set of NPM packages in target definition: " + ",".join([ + # dep.label + # for dep in direct_npm_packages + # ])) + # extended_npm_packages = depset( + # direct = direct_npm_packages, + # transitive = [ + # dep[JsLibraryInfo].npm_packages + # for dep in ctx.attr.deps + # if JsLibraryInfo in dep + # ], + # ) + # npm_packages_list = extended_npm_packages.to_list() + # if len(npm_packages_list) > 1: + # fail("Found more than one set of NPM packages through dependencies: " + ",".join([ + # dep.label + # for dep in npm_packages_list + # ])) + # # If we depend on an npm_packages target, we'll use its node_modules + # # directory to find modules. Otherwise, we'll use an empty node_modules + # # directory. + # npm_packages = ( + # npm_packages_list[0] if len(npm_packages_list) == 1 + # else ctx.attr._empty_npm_packages + # ) + # # Gather all internal deps (other ts_library rules). + # internal_deps = depset( + # direct = [ + # dep + # for dep in ctx.attr.deps + # if JsLibraryInfo in dep + # ], + # transitive = [ + # dep[JsLibraryInfo].internal_deps + # for dep in ctx.attr.deps + # if JsLibraryInfo in dep + # ], + # ) + + # # Create two directories that contain: + # # - source files (including all internal dependencies) + # # - node_modules (symlinked to installed external dependencies directory) + + # # First version includes all dependencies' TypeScript definitions, which + # # requires compiling everything up the tree (slow). Necessary to be able + # # to compile TypeScript, including type verification. + # _ts_library_create_full_src( + # ctx, + # internal_deps, + # npm_packages, + # ctx.outputs.compilation_src_dir, + # True, + # ) + + # # Second version only includes dependencies' transpiled JavaScript code, + # # which is a lot faster but does not do any type checking. + # _ts_library_create_full_src( + # ctx, + # internal_deps, + # npm_packages, + # ctx.outputs.transpilation_src_dir, + # False, + # ) + + # # Compile the directory with `tsc` (slower but stricter). + # _ts_library_compile( + # ctx, + # internal_deps, + # npm_packages, + # ) + + # # Transpile the directory with `tsc` (faster, no type checking). + # _ts_library_transpile( + # ctx, + # internal_deps, + # npm_packages, + # ) + + # return [ + # JsLibraryInfo( + # build_file_path = ctx.build_file_path, + # javascript_source_files = [_compiled_extension(f.path) for f in ctx.files.srcs], + # compiled_javascript_dir = ctx.outputs.transpiled_dir, + # internal_deps = internal_deps, + # npm_packages = extended_npm_packages, + # npm_packages_installed_dir = npm_packages[NpmPackagesInfo].installed_dir, + # ), + # TsLibraryInfo( + # original_typescript_dir = ctx.outputs.compilation_src_dir, + # compiled_typescript_dir = ctx.outputs.compiled_dir, + # typescript_source_files = [f.path for f in ctx.files.srcs], + # ), + # ] def _compiled_extension(path): - if path.endswith('.tsx'): - return path[:-4] + '.js' - elif path.endswith('.ts'): - return path[:-3] + '.js' - else: - return path + if path.endswith(".tsx"): + return path[:-4] + ".js" + elif path.endswith(".ts"): + return path[:-3] + ".js" + else: + return path -def _ts_library_create_full_src(ctx, internal_deps, npm_packages, output_dir, for_compilation): - ctx.actions.run( - inputs = [ - ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, - ctx.file._ts_library_create_full_src_script, - npm_packages[NpmPackagesInfo].installed_dir, - ctx.file.tsconfig, - ] + [ - d[TsLibraryInfo].original_typescript_dir - if for_compilation and TsLibraryInfo in d - else d[JsLibraryInfo].compiled_javascript_dir - for d in internal_deps - ] + ctx.files.srcs, - outputs = [output_dir], - executable = ctx.file._internal_nodejs, - env = { - "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules" - }, - arguments = [ - # Run `node create_full_src.js`. - ctx.file._ts_library_create_full_src_script.path, - # Label of the build target (for helpful errors). - "//" + ctx.label.package + ":" + ctx.label.name, - # Directory containing node_modules/ with all external NPM packages - # installed. - npm_packages[NpmPackagesInfo].installed_dir.path, - # tsconfig.json path. - ctx.file.tsconfig.path, - # Source directories of the ts_library targets we depend on. - ("|".join([ - (";".join( - d[TsLibraryInfo].typescript_source_files - if for_compilation and TsLibraryInfo in d - else d[JsLibraryInfo].javascript_source_files - )) + - ":" + - ( - d[TsLibraryInfo].original_typescript_dir.path - if for_compilation and TsLibraryInfo in d - else d[JsLibraryInfo].compiled_javascript_dir.path - ) - for d in internal_deps - ])), - # List of source files, which will be processed ("import" statements - # automatically replaced) and copied into the new directory. - ("|".join([ - f.path for f in ctx.files.srcs - ])), - # Directory in which to place the result. - output_dir.path, - ], - ) +# def _ts_library_create_full_src(ctx, internal_deps, npm_packages, output_dir, for_compilation): +# ctx.actions.run( +# inputs = [ +# ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, +# ctx.file._ts_library_create_full_src_script, +# npm_packages[NpmPackagesInfo].installed_dir, +# ctx.file.tsconfig, +# ] + [ +# d[TsLibraryInfo].original_typescript_dir if for_compilation and TsLibraryInfo in d else d[JsLibraryInfo].compiled_javascript_dir +# for d in internal_deps +# ] + ctx.files.srcs, +# outputs = [output_dir], +# executable = ctx.file._internal_nodejs, +# env = { +# "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", +# }, +# arguments = [ +# # Run `node create_full_src.js`. +# ctx.file._ts_library_create_full_src_script.path, +# # Label of the build target (for helpful errors). +# "//" + ctx.label.package + ":" + ctx.label.name, +# # Directory containing node_modules/ with all external NPM packages +# # installed. +# npm_packages[NpmPackagesInfo].installed_dir.path, +# # tsconfig.json path. +# ctx.file.tsconfig.path, +# # Source directories of the ts_library targets we depend on. +# ("|".join([ +# (";".join( +# d[TsLibraryInfo].typescript_source_files if for_compilation and TsLibraryInfo in d else d[JsLibraryInfo].javascript_source_files, +# )) + +# ":" + +# ( +# d[TsLibraryInfo].original_typescript_dir.path if for_compilation and TsLibraryInfo in d else d[JsLibraryInfo].compiled_javascript_dir.path +# ) +# for d in internal_deps +# ])), +# # List of source files, which will be processed ("import" statements +# # automatically replaced) and copied into the new directory. +# ("|".join([ +# f.path +# for f in ctx.files.srcs +# ])), +# # Directory in which to place the result. +# output_dir.path, +# ], +# ) -def _ts_library_compile(ctx, internal_deps, npm_packages): - ctx.actions.run( - inputs = [ - ctx.file._ts_library_compile_script, - ctx.outputs.compilation_src_dir, - ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, - npm_packages[NpmPackagesInfo].installed_dir, - ] + [ - d[TsLibraryInfo].original_typescript_dir - if TsLibraryInfo in d - else d[JsLibraryInfo].compiled_javascript_dir - for d in internal_deps - ], - outputs = [ctx.outputs.compiled_dir], - executable = ctx.file._internal_nodejs, - env = { - "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules" - }, - arguments = [ - # Run `node ts_library/compile.js`. - ctx.file._ts_library_compile_script.path, - # Directory in which the source code as well as tsconfig.json can be found. - ctx.outputs.compilation_src_dir.path, - # Directory in which to generate the compiled JavaScript and TypeScript - # definitions. - ctx.outputs.compiled_dir.path, - ], - ) +# def _ts_library_compile(ctx, internal_deps, npm_packages): +# ctx.actions.run( +# inputs = [ +# ctx.file._ts_library_compile_script, +# ctx.outputs.compilation_src_dir, +# ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, +# npm_packages[NpmPackagesInfo].installed_dir, +# ] + [ +# d[TsLibraryInfo].original_typescript_dir if TsLibraryInfo in d else d[JsLibraryInfo].compiled_javascript_dir +# for d in internal_deps +# ], +# outputs = [ctx.outputs.compiled_dir], +# executable = ctx.file._internal_nodejs, +# env = { +# "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", +# }, +# arguments = [ +# # Run `node ts_library/compile.js`. +# ctx.file._ts_library_compile_script.path, +# # Directory in which the source code as well as tsconfig.json can be found. +# ctx.outputs.compilation_src_dir.path, +# # Directory in which to generate the compiled JavaScript and TypeScript +# # definitions. +# ctx.outputs.compiled_dir.path, +# ], +# ) -def _ts_library_transpile(ctx, internal_deps, npm_packages): - ctx.actions.run( - inputs = [ - ctx.file._ts_library_transpile_script, - ctx.outputs.transpilation_src_dir, - ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, - npm_packages[NpmPackagesInfo].installed_dir, - ] + [ - d[JsLibraryInfo].compiled_javascript_dir - for d in internal_deps - ], - outputs = [ctx.outputs.transpiled_dir], - executable = ctx.file._internal_nodejs, - env = { - "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules" - }, - arguments = [ - # Run `node ts_library/transpile.js`. - ctx.file._ts_library_transpile_script.path, - # Directory in which the source code as well as tsconfig.json can be found. - ctx.outputs.transpilation_src_dir.path, - # Directory in which to generate the transpiled JavaScript and TypeScript - # definitions. - ctx.outputs.transpiled_dir.path, - ], - ) +# def _ts_library_transpile(ctx, internal_deps, npm_packages): +# ctx.actions.run( +# inputs = [ +# ctx.file._ts_library_transpile_script, +# ctx.outputs.transpilation_src_dir, +# ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, +# npm_packages[NpmPackagesInfo].installed_dir, +# ] + [ +# d[JsLibraryInfo].compiled_javascript_dir +# for d in internal_deps +# ], +# outputs = [ctx.outputs.transpiled_dir], +# executable = ctx.file._internal_nodejs, +# env = { +# "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", +# }, +# arguments = [ +# # Run `node ts_library/transpile.js`. +# ctx.file._ts_library_transpile_script.path, +# # Directory in which the source code as well as tsconfig.json can be found. +# ctx.outputs.transpilation_src_dir.path, +# # Directory in which to generate the transpiled JavaScript and TypeScript +# # definitions. +# ctx.outputs.transpiled_dir.path, +# ], +# ) ts_library = rule( - implementation=_ts_library_impl, - attrs = { - "srcs": attr.label_list( - allow_files = True, - mandatory = True, - ), - "deps": attr.label_list( - providers = [ - [JsLibraryInfo], - [NpmPackagesInfo], - ], - default = [], - ), - "tsconfig": attr.label( - allow_files = [".json"], - single_file = True, - default = Label("//internal/ts_library:default_tsconfig.json"), - ), - "_internal_nodejs": attr.label( - allow_files = True, - single_file = True, - default = Label("@nodejs//:node"), - ), - "_internal_packages": attr.label( - default = Label("//internal:packages"), - ), - "_ts_library_create_full_src_script": attr.label( - allow_files = True, - single_file = True, - default = Label("//internal/ts_library:create_full_src.js"), - ), - "_ts_library_compile_script": attr.label( - allow_files = True, - single_file = True, - default = Label("//internal/ts_library:compile.js"), - ), - "_ts_library_transpile_script": attr.label( - allow_files = True, - single_file = True, - default = Label("//internal/ts_library:transpile.js"), - ), - "_empty_npm_packages": attr.label( - default = Label("//internal/npm_packages/empty:packages"), - ), - }, - outputs = { - "compilation_src_dir": "%{name}_compilation_src", - "compiled_dir": "%{name}_compiled", - "transpilation_src_dir": "%{name}_transpilation_src", - "transpiled_dir": "%{name}_transpiled", - }, + implementation = _ts_library_impl, + attrs = dict(JS_LIBRARY_ATTRIBUTES, **{ + "tsconfig": attr.label( + allow_files = [".json"], + single_file = True, + default = Label("//internal/ts_library:default_tsconfig.json"), + ), + "_ts_library_create_full_src_script": attr.label( + allow_files = True, + single_file = True, + default = Label("//internal/ts_library:create_full_src.js"), + ), + "_ts_library_compile_script": attr.label( + allow_files = True, + single_file = True, + default = Label("//internal/ts_library:compile.js"), + ), + "_ts_library_transpile_script": attr.label( + allow_files = True, + single_file = True, + default = Label("//internal/ts_library:transpile.js"), + ), + "_ts_config_genscript": attr.label( + allow_files = True, + single_file = True, + default = Label("//internal/ts_library:tsconfig.gen.js"), + ) + }), + outputs = { + "compilation_src_dir": "%{name}_compilation_src", + "compiled_dir": "%{name}_compiled", + # "transpilation_src_dir": "%{name}_transpilation_src", + # "transpiled_dir": "%{name}_transpiled", + }, ) diff --git a/internal/ts_library/tsconfig.gen.js b/internal/ts_library/tsconfig.gen.js new file mode 100644 index 0000000..36123c3 --- /dev/null +++ b/internal/ts_library/tsconfig.gen.js @@ -0,0 +1,53 @@ +const fs = require("fs-extra"); +const path = require("path"); + +const DEFAULT_TSCONFIG = { + compilerOptions: { + target: "es2015", + module: "es2015", + moduleResolution: "node", + declaration: true, + strict: true, + noImplicitAny: true, + strictNullChecks: true, + strictFunctionTypes: true, + strictPropertyInitialization: true, + noImplicitThis: true, + alwaysStrict: true, + jsx: "react", + allowSyntheticDefaultImports: true, + baseUrl: ".", + paths: { + "@/*": ["src/*"] + } + } +}; + +module.exports = async ({ package, into, inputs }) => { + const inputFiles = inputs; + if (inputFiles.length > 1) { + throw new Error(`Got too many files for tsconfig generation ${inputFiles}`); + } + const tsconfigInput = inputFiles[0] + ? JSON.parse(inputFiles[0].body) + : DEFAULT_TSCONFIG; + + const compilerOptions = {}; + Object.assign(compilerOptions, tsconfigInput.compilerOptions || {}); + Object.assign(compilerOptions, { + moduleResolution: "node", + declaration: true, + rootDir: "." + }); + delete compilerOptions.allowJs; + return [ + { + path: "tsconfig.json", + body: JSON.stringify({ + compilerOptions, + exclude: ["node_modules"], + include: [`${package.path}/**/*.ts`, `${package.path}/**/*.tsx`] + }) + } + ]; +}; diff --git a/internal/yarn.lock b/internal/yarn.lock index ff476ba..2365732 100644 --- a/internal/yarn.lock +++ b/internal/yarn.lock @@ -2185,6 +2185,10 @@ get-value@^2.0.3, get-value@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" +getopts@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.1.tgz#68120d77abf420e1ade52291977ce050f33ce54e" + glob-parent@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-3.1.0.tgz#9e6af6299d8d3bd2bd40430832bd113df906c5ae" From df6ff06ccadb1cd89c8cfcd542bb588e13cb4cd3 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 11 Oct 2018 06:18:40 -0700 Subject: [PATCH 02/43] Updating rules_nodejs dependency --- WORKSPACE | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 184e484..578ee45 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,15 +3,11 @@ workspace(name = "bazel_javascript") git_repository( name = "build_bazel_rules_nodejs", remote = "https://github.com/bazelbuild/rules_nodejs.git", - tag = "0.14.2", + tag = "0.15.0", ) -# Required by build_bazel_rules_nodejs. -git_repository( - name = "bazel_skylib", - remote = "https://github.com/bazelbuild/bazel-skylib.git", - tag = "0.5.0", -) +load("@build_bazel_rules_nodejs//:package.bzl", "rules_nodejs_dependencies") +rules_nodejs_dependencies() load("@build_bazel_rules_nodejs//:defs.bzl", "node_repositories") From 7d664dfd8ffb6f3c78e2520b69e87e3f02360263 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 11 Oct 2018 06:19:05 -0700 Subject: [PATCH 03/43] Adding prettier configuration --- .prettierrc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..1ca87ab --- /dev/null +++ b/.prettierrc @@ -0,0 +1,3 @@ +{ + "singleQuote": false +} From 70313be6462ecb864e51de13634556d9c5a9a75c Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 11 Oct 2018 06:21:23 -0700 Subject: [PATCH 04/43] Using non-relative imports in node app example --- examples/node-typescript-app/WORKSPACE | 10 ++++-- .../libs/shared-package/BUILD.bazel | 1 + .../libs/shared-package/common.ts | 3 ++ .../libs/shared-package/greeter.ts | 4 ++- examples/node-typescript-app/package.json | 1 + .../services/my-service/server/server.ts | 2 +- examples/node-typescript-app/tsconfig.json | 6 +++- examples/node-typescript-app/yarn.lock | 36 ++++++++++++++++++- 8 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 examples/node-typescript-app/libs/shared-package/common.ts diff --git a/examples/node-typescript-app/WORKSPACE b/examples/node-typescript-app/WORKSPACE index f3b187d..0b25b6c 100644 --- a/examples/node-typescript-app/WORKSPACE +++ b/examples/node-typescript-app/WORKSPACE @@ -15,7 +15,7 @@ local_repository( git_repository( name = "build_bazel_rules_nodejs", remote = "https://github.com/bazelbuild/rules_nodejs.git", - tag = "0.14.2", + tag = "0.15.0", ) # Required by build_bazel_rules_nodejs. @@ -25,12 +25,18 @@ git_repository( tag = "0.5.0", ) -load("@build_bazel_rules_nodejs//:defs.bzl", "node_repositories") +load("@build_bazel_rules_nodejs//:defs.bzl", "node_repositories", "yarn_install") node_repositories( package_json = [], ) +yarn_install( + name = "npm", + package_json = "//:package.json", + yarn_lock = "//:yarn.lock", +) + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( diff --git a/examples/node-typescript-app/libs/shared-package/BUILD.bazel b/examples/node-typescript-app/libs/shared-package/BUILD.bazel index 382a702..bf6d3ce 100644 --- a/examples/node-typescript-app/libs/shared-package/BUILD.bazel +++ b/examples/node-typescript-app/libs/shared-package/BUILD.bazel @@ -6,4 +6,5 @@ ts_library( name = "shared-package", srcs = glob(["**/*.ts"]), tsconfig = "//:tsconfig.json", + deps = ["//:packages"], ) diff --git a/examples/node-typescript-app/libs/shared-package/common.ts b/examples/node-typescript-app/libs/shared-package/common.ts new file mode 100644 index 0000000..e14b069 --- /dev/null +++ b/examples/node-typescript-app/libs/shared-package/common.ts @@ -0,0 +1,3 @@ +export function tryMe(aString: String) { + return `I am trying: ${aString}`; +} diff --git a/examples/node-typescript-app/libs/shared-package/greeter.ts b/examples/node-typescript-app/libs/shared-package/greeter.ts index ae9dfa6..8a2c619 100644 --- a/examples/node-typescript-app/libs/shared-package/greeter.ts +++ b/examples/node-typescript-app/libs/shared-package/greeter.ts @@ -1,3 +1,5 @@ +import chalk from "chalk"; + export function greet(name: string): string { - return `Hello, ${name}`; + return `Hello, ${chalk.red(name)}`; } diff --git a/examples/node-typescript-app/package.json b/examples/node-typescript-app/package.json index b72b46e..23c18f5 100644 --- a/examples/node-typescript-app/package.json +++ b/examples/node-typescript-app/package.json @@ -5,6 +5,7 @@ "@types/koa": "^2.0.46", "@types/node": "^10.7.1", "@types/node-sass": "^3.10.32", + "chalk": "^2.4.1", "koa": "^2.5.2", "node-sass": "^4.9.3", "source-map-support": "^0.5.9" diff --git a/examples/node-typescript-app/services/my-service/server/server.ts b/examples/node-typescript-app/services/my-service/server/server.ts index 5eff999..8022526 100644 --- a/examples/node-typescript-app/services/my-service/server/server.ts +++ b/examples/node-typescript-app/services/my-service/server/server.ts @@ -1,6 +1,6 @@ import * as Koa from "koa"; import * as sass from "node-sass"; -import { greet } from "../../../libs/shared-package/greeter"; +import { greet } from "shared-package/greeter"; const app = new Koa(); diff --git a/examples/node-typescript-app/tsconfig.json b/examples/node-typescript-app/tsconfig.json index c76b808..79d219d 100644 --- a/examples/node-typescript-app/tsconfig.json +++ b/examples/node-typescript-app/tsconfig.json @@ -13,6 +13,10 @@ "noImplicitThis": true, "alwaysStrict": true, "jsx": "react", - "allowSyntheticDefaultImports": false + "allowSyntheticDefaultImports": false, + "baseUrl": ".", + "paths": { + "*": ["*", "libs/*"] + } } } diff --git a/examples/node-typescript-app/yarn.lock b/examples/node-typescript-app/yarn.lock index dd23c95..ae92e32 100644 --- a/examples/node-typescript-app/yarn.lock +++ b/examples/node-typescript-app/yarn.lock @@ -135,6 +135,12 @@ ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + dependencies: + color-convert "^1.9.0" + any-promise@^1.1.0: version "1.3.0" resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" @@ -247,6 +253,14 @@ chalk@^1.1.1: strip-ansi "^3.0.0" supports-color "^2.0.0" +chalk@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.1.tgz#18c49ab16a037b6eb0152cc83e3471338215b66e" + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + cliui@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" @@ -263,6 +277,16 @@ code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + dependencies: + color-name "1.1.3" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + combined-stream@1.0.6, combined-stream@~1.0.5, combined-stream@~1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.6.tgz#723e7df6e801ac5613113a7e445a9b69cb632818" @@ -370,7 +394,7 @@ escape-html@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" -escape-string-regexp@^1.0.2: +escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" @@ -520,6 +544,10 @@ has-ansi@^2.0.0: dependencies: ansi-regex "^2.0.0" +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" @@ -1276,6 +1304,12 @@ supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + dependencies: + has-flag "^3.0.0" + tar@^2.0.0: version "2.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1" From 96d1baa11455114862a5a9388351fb03b7c4febd Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 11 Oct 2018 06:26:15 -0700 Subject: [PATCH 05/43] Removing some cruft --- .../create_source_dir/create_source_dir.js | 85 ++----------------- 1 file changed, 7 insertions(+), 78 deletions(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index f19e272..ae609e8 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -1,10 +1,6 @@ const fs = require("fs-extra"); const path = require("path"); -const { - BazelAction, - safeSymlink, - ensureArray -} = require("../run_js/BazelAction"); +const { BazelAction, ensureArray } = require("../run_js/BazelAction"); /** * Action for creating a directory with the passed in files for symlinking and @@ -91,83 +87,16 @@ BazelAction( } }; - console.log(`Current Target: ${current_target}`); - console.log(`Target Source Path: ${from}`); - console.log(`Copying into: ${into}`); - console.log(`Running in [${process.cwd()}]`); - console.log(`Environment is: ${JSON.stringify(process.env, null, 2)}`); - console.log(`Will create [${into}] for populating`); + // console.log(`Current Target: ${current_target}`); + // console.log(`Target Source Path: ${from}`); + // console.log(`Copying into: ${into}`); + // console.log(`Running in [${process.cwd()}]`); + // console.log(`Environment is: ${JSON.stringify(process.env, null, 2)}`); + // console.log(`Will create [${into}] for populating`); for (const source of sources) { await populateFiles(source); } - - // const internalDeps = joinedInternalDeps.split("|"); - // const srcs = joinedSrcs.split("|"); - - // fs.mkdirSync(into); - // safeSymlink( - // path.join(installedNpmPackagesDir, "node_modules"), - // path.join(destinationDir, "node_modules") - // ); - - // // Copy every internal dependency into the appropriate location. - // for (const internalDep of internalDeps) { - // if (!internalDep) { - // continue; - // } - // const [joinedSrcs, compiledDir] = internalDep.split(":"); - // const srcs = joinedSrcs.split(";"); - // for (const src of srcs) { - // if (!src) { - // continue; - // } - // safeSymlink( - // path.join(compiledDir, src), - // path.join(destinationDir, src) - // ); - // } - // } - - // // Extract compiler options from tsconfig.json, overriding anything other - // // than compiler options. - // const originalTsConfig = JSON.parse(fs.readFileSync(tsconfigPath, "utf8")); - - // // Copy source code and update import statements in this target's sources. - // for (const src of srcs) { - // if (!src) { - // continue; - // } - // if (!fs.existsSync(src)) { - // console.error(` - // Missing file ${src} required by ${targetLabel}. - // `); - // process.exit(1); - // } - // const destinationFilePath = path.join(destinationDir, src); - // fs.ensureDirSync(path.dirname(destinationFilePath)); - // safeSymlink(src, destinationFilePath); - // } - - // const compilerOptions = {}; - // Object.assign(compilerOptions, originalTsConfig.compilerOptions || {}); - // Object.assign(compilerOptions, { - // moduleResolution: "node", - // declaration: true, - // rootDir: "." - // }); - // fs.writeFileSync( - // path.join(destinationDir, "tsconfig.json"), - // JSON.stringify( - // { - // compilerOptions, - // files: srcs.filter(src => src.endsWith(".ts") || src.endsWith(".tsx")) - // }, - // null, - // 2 - // ), - // "utf8" - // ); } ); From 19b70f92fe33f3bac2ce010a1944306b60de0ffb Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 11 Oct 2018 12:18:38 -0700 Subject: [PATCH 06/43] Removing print statements --- internal/common/actions/create_source_dir/action.bzl | 3 --- internal/common/context.bzl | 2 -- 2 files changed, 5 deletions(-) diff --git a/internal/common/actions/create_source_dir/action.bzl b/internal/common/actions/create_source_dir/action.bzl index 0a2a5cb..a9d5ede 100644 --- a/internal/common/actions/create_source_dir/action.bzl +++ b/internal/common/actions/create_source_dir/action.bzl @@ -23,9 +23,6 @@ def create_source_dir(js, js_source, create_dir): script_args = js.script_args(js, js._create_source_dir_js) script_args.add("--into", create_dir) script_args.add("--from", js.package_path) - print(library.all_sources) - print(library.module_paths) - print(library.all_modules) for gen_script in gen_scripts: if type(gen_script) == type(""): diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 77f4363..24d2a8f 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -148,7 +148,6 @@ def _js_library_info(js, attr = None): attr = js.attr # The srcs should contain what has been explicitly added for a rule - srcs_attr = getattr(attr, "srcs", []) src_files = js._ctx.files.srcs # The deps is list of labels that should have providers that we can get sources from @@ -161,7 +160,6 @@ def _js_library_info(js, attr = None): # Iterate through the deps to add them to their correct JsLibrary attributes for dep in deps_attr: - print(dep) if JsLibraryInfo in dep: dep_js_library = dep[JsLibraryInfo] # The dependency is another JsLibrary From ea9dc5806ea7202e2ed603daae4b7fa28e49e7a3 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 18 Oct 2018 14:19:47 -0700 Subject: [PATCH 07/43] Adding generation of JsModuleInfo --- internal/common/context.bzl | 21 +++- internal/ts_library/rule.bzl | 223 +++-------------------------------- 2 files changed, 31 insertions(+), 213 deletions(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 24d2a8f..8ebd79f 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -33,9 +33,9 @@ JsLibraryInfo = provider( "all_sources", # Entire paths to add to npm modules (eg. a node_modules path) "module_paths", - # Depset of JsModule that this library depends on + # Modules that are depended upon directly "modules", - # All modules that this library depends on + # Transitive module dependencies "all_modules", ], ) @@ -56,7 +56,7 @@ JsSourceInfo = provider(fields = [ """Source that will be used either directly or for transpilation to javascript """ -JsModule = provider( +JsModuleInfo = provider( fields = [ # The root of the workspace "workspace_name", @@ -171,8 +171,12 @@ def _js_library_info(js, attr = None): elif NpmPackagesInfo in dep: # The dependency is a node_modules directory installed by npm_packages module_paths.append(dep[NpmPackagesInfo].installed_dir) - + if JsModuleInfo in dep: + dep_js_module = dep[JsModuleInfo] + direct_modules.append(dep_js_module) + transitive_modules.append(dep_js_module.all_modules) + all_src_files = depset( direct = src_files, transitive = transitive_sources, @@ -204,6 +208,13 @@ def _library_to_source_info(js, library, gen_scripts = None): library = library, ) +def _library_to_module_info(js, library, module_root, module_name): + return JsModuleInfo( + workspace_name = js.workspace_name, + all_modules = library.all_modules, + module_root = module_root, + module_name = module_name, + ) def _script_args(js, script_file): """Create Args object that can be used with js.run_js() @@ -264,6 +275,7 @@ def js_context(ctx, attr = None): # Fields workspace_name = ctx.workspace_name, package_path = ctx.label.package, + module_name = getattr(attr, "module_name", None), _ctx = ctx, _internal_nodejs = _internal_nodejs, _internal_packages = _internal_packages, @@ -280,4 +292,5 @@ def js_context(ctx, attr = None): script_args = _script_args, library_info = _js_library_info, library_to_source_info = _library_to_source_info, + library_to_module_info = _library_to_module_info, ) diff --git a/internal/ts_library/rule.bzl b/internal/ts_library/rule.bzl index e354b4b..a39427e 100644 --- a/internal/ts_library/rule.bzl +++ b/internal/ts_library/rule.bzl @@ -13,17 +13,30 @@ TsLibraryInfo = provider(fields = [ def _ts_library_impl(ctx): js = js_context(ctx) + providers = [] # for i, d in enumerate(ctx.attr.deps): # print(" {}. label = {}".format(i + 1, d.label)) # print(" files = " + str([f.path for f in d.files.to_list()])) js_library = js.library_info(js) + providers.append(js_library) + js_source = js.library_to_source_info(js, js_library, gen_scripts = [ [ctx.file._ts_config_genscript, ctx.file.tsconfig] ]) + providers.append(js_source) js.create_source_dir(js, js_source, ctx.outputs.compilation_src_dir) + if js.module_name: + js_module = js.library_to_module_info( + js, + js_library, + module_name = js.module_name, + module_root = ctx.outputs.compiled_dir, + ) + providers.append(js_module) + compile_args = js.script_args(js, ctx.file._ts_library_compile_script.path) compile_args.add("--project", ctx.outputs.compilation_src_dir) compile_args.add("--outDir", ctx.outputs.compiled_dir) @@ -41,113 +54,7 @@ def _ts_library_impl(ctx): script_args = compile_args, ) - return [ - js_library, - js_source, - ] - - # Ensure that we depend on at most one npm_packages, since we don't want to - # have conflicting package versions coming from separate node_modules - # directories. - # direct_npm_packages = [ - # dep - # for dep in ctx.attr.deps - # if NpmPackagesInfo in dep - # ] - # if len(direct_npm_packages) > 1: - # fail("Found more than one set of NPM packages in target definition: " + ",".join([ - # dep.label - # for dep in direct_npm_packages - # ])) - # extended_npm_packages = depset( - # direct = direct_npm_packages, - # transitive = [ - # dep[JsLibraryInfo].npm_packages - # for dep in ctx.attr.deps - # if JsLibraryInfo in dep - # ], - # ) - # npm_packages_list = extended_npm_packages.to_list() - # if len(npm_packages_list) > 1: - # fail("Found more than one set of NPM packages through dependencies: " + ",".join([ - # dep.label - # for dep in npm_packages_list - # ])) - # # If we depend on an npm_packages target, we'll use its node_modules - # # directory to find modules. Otherwise, we'll use an empty node_modules - # # directory. - # npm_packages = ( - # npm_packages_list[0] if len(npm_packages_list) == 1 - # else ctx.attr._empty_npm_packages - # ) - # # Gather all internal deps (other ts_library rules). - # internal_deps = depset( - # direct = [ - # dep - # for dep in ctx.attr.deps - # if JsLibraryInfo in dep - # ], - # transitive = [ - # dep[JsLibraryInfo].internal_deps - # for dep in ctx.attr.deps - # if JsLibraryInfo in dep - # ], - # ) - - # # Create two directories that contain: - # # - source files (including all internal dependencies) - # # - node_modules (symlinked to installed external dependencies directory) - - # # First version includes all dependencies' TypeScript definitions, which - # # requires compiling everything up the tree (slow). Necessary to be able - # # to compile TypeScript, including type verification. - # _ts_library_create_full_src( - # ctx, - # internal_deps, - # npm_packages, - # ctx.outputs.compilation_src_dir, - # True, - # ) - - # # Second version only includes dependencies' transpiled JavaScript code, - # # which is a lot faster but does not do any type checking. - # _ts_library_create_full_src( - # ctx, - # internal_deps, - # npm_packages, - # ctx.outputs.transpilation_src_dir, - # False, - # ) - - # # Compile the directory with `tsc` (slower but stricter). - # _ts_library_compile( - # ctx, - # internal_deps, - # npm_packages, - # ) - - # # Transpile the directory with `tsc` (faster, no type checking). - # _ts_library_transpile( - # ctx, - # internal_deps, - # npm_packages, - # ) - - # return [ - # JsLibraryInfo( - # build_file_path = ctx.build_file_path, - # javascript_source_files = [_compiled_extension(f.path) for f in ctx.files.srcs], - # compiled_javascript_dir = ctx.outputs.transpiled_dir, - # internal_deps = internal_deps, - # npm_packages = extended_npm_packages, - # npm_packages_installed_dir = npm_packages[NpmPackagesInfo].installed_dir, - # ), - # TsLibraryInfo( - # original_typescript_dir = ctx.outputs.compilation_src_dir, - # compiled_typescript_dir = ctx.outputs.compiled_dir, - # typescript_source_files = [f.path for f in ctx.files.srcs], - # ), - # ] + return providers def _compiled_extension(path): if path.endswith(".tsx"): @@ -157,108 +64,6 @@ def _compiled_extension(path): else: return path -# def _ts_library_create_full_src(ctx, internal_deps, npm_packages, output_dir, for_compilation): -# ctx.actions.run( -# inputs = [ -# ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, -# ctx.file._ts_library_create_full_src_script, -# npm_packages[NpmPackagesInfo].installed_dir, -# ctx.file.tsconfig, -# ] + [ -# d[TsLibraryInfo].original_typescript_dir if for_compilation and TsLibraryInfo in d else d[JsLibraryInfo].compiled_javascript_dir -# for d in internal_deps -# ] + ctx.files.srcs, -# outputs = [output_dir], -# executable = ctx.file._internal_nodejs, -# env = { -# "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", -# }, -# arguments = [ -# # Run `node create_full_src.js`. -# ctx.file._ts_library_create_full_src_script.path, -# # Label of the build target (for helpful errors). -# "//" + ctx.label.package + ":" + ctx.label.name, -# # Directory containing node_modules/ with all external NPM packages -# # installed. -# npm_packages[NpmPackagesInfo].installed_dir.path, -# # tsconfig.json path. -# ctx.file.tsconfig.path, -# # Source directories of the ts_library targets we depend on. -# ("|".join([ -# (";".join( -# d[TsLibraryInfo].typescript_source_files if for_compilation and TsLibraryInfo in d else d[JsLibraryInfo].javascript_source_files, -# )) + -# ":" + -# ( -# d[TsLibraryInfo].original_typescript_dir.path if for_compilation and TsLibraryInfo in d else d[JsLibraryInfo].compiled_javascript_dir.path -# ) -# for d in internal_deps -# ])), -# # List of source files, which will be processed ("import" statements -# # automatically replaced) and copied into the new directory. -# ("|".join([ -# f.path -# for f in ctx.files.srcs -# ])), -# # Directory in which to place the result. -# output_dir.path, -# ], -# ) - -# def _ts_library_compile(ctx, internal_deps, npm_packages): -# ctx.actions.run( -# inputs = [ -# ctx.file._ts_library_compile_script, -# ctx.outputs.compilation_src_dir, -# ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, -# npm_packages[NpmPackagesInfo].installed_dir, -# ] + [ -# d[TsLibraryInfo].original_typescript_dir if TsLibraryInfo in d else d[JsLibraryInfo].compiled_javascript_dir -# for d in internal_deps -# ], -# outputs = [ctx.outputs.compiled_dir], -# executable = ctx.file._internal_nodejs, -# env = { -# "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", -# }, -# arguments = [ -# # Run `node ts_library/compile.js`. -# ctx.file._ts_library_compile_script.path, -# # Directory in which the source code as well as tsconfig.json can be found. -# ctx.outputs.compilation_src_dir.path, -# # Directory in which to generate the compiled JavaScript and TypeScript -# # definitions. -# ctx.outputs.compiled_dir.path, -# ], -# ) - -# def _ts_library_transpile(ctx, internal_deps, npm_packages): -# ctx.actions.run( -# inputs = [ -# ctx.file._ts_library_transpile_script, -# ctx.outputs.transpilation_src_dir, -# ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, -# npm_packages[NpmPackagesInfo].installed_dir, -# ] + [ -# d[JsLibraryInfo].compiled_javascript_dir -# for d in internal_deps -# ], -# outputs = [ctx.outputs.transpiled_dir], -# executable = ctx.file._internal_nodejs, -# env = { -# "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", -# }, -# arguments = [ -# # Run `node ts_library/transpile.js`. -# ctx.file._ts_library_transpile_script.path, -# # Directory in which the source code as well as tsconfig.json can be found. -# ctx.outputs.transpilation_src_dir.path, -# # Directory in which to generate the transpiled JavaScript and TypeScript -# # definitions. -# ctx.outputs.transpiled_dir.path, -# ], -# ) - ts_library = rule( implementation = _ts_library_impl, attrs = dict(JS_LIBRARY_ATTRIBUTES, **{ From 5d1111e507b1d5bbb0bf0fae7bb84b310ccc0658 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 18 Oct 2018 15:11:21 -0700 Subject: [PATCH 08/43] Removing unused script --- internal/ts_library/create_full_src.js | 79 -------------------------- 1 file changed, 79 deletions(-) delete mode 100644 internal/ts_library/create_full_src.js diff --git a/internal/ts_library/create_full_src.js b/internal/ts_library/create_full_src.js deleted file mode 100644 index b9ffbbb..0000000 --- a/internal/ts_library/create_full_src.js +++ /dev/null @@ -1,79 +0,0 @@ -const fs = require("fs-extra"); -const path = require("path"); -const { safeSymlink } = require("../common/symlink"); - -const [ - nodePath, - scriptPath, - targetLabel, - installedNpmPackagesDir, - tsconfigPath, - joinedInternalDeps, - joinedSrcs, - destinationDir -] = process.argv; - -const internalDeps = joinedInternalDeps.split("|"); -const srcs = joinedSrcs.split("|"); - -fs.mkdirSync(destinationDir); -safeSymlink( - path.join(installedNpmPackagesDir, "node_modules"), - path.join(destinationDir, "node_modules") -); - -// Copy every internal dependency into the appropriate location. -for (const internalDep of internalDeps) { - if (!internalDep) { - continue; - } - const [joinedSrcs, compiledDir] = internalDep.split(":"); - const srcs = joinedSrcs.split(";"); - for (const src of srcs) { - if (!src) { - continue; - } - safeSymlink(path.join(compiledDir, src), path.join(destinationDir, src)); - } -} - -// Extract compiler options from tsconfig.json, overriding anything other -// than compiler options. -const originalTsConfig = JSON.parse(fs.readFileSync(tsconfigPath, "utf8")); - -// Copy source code and update import statements in this target's sources. -for (const src of srcs) { - if (!src) { - continue; - } - if (!fs.existsSync(src)) { - console.error(` -Missing file ${src} required by ${targetLabel}. -`); - process.exit(1); - } - const destinationFilePath = path.join(destinationDir, src); - fs.ensureDirSync(path.dirname(destinationFilePath)); - safeSymlink(src, destinationFilePath); -} - -const compilerOptions = {}; -Object.assign(compilerOptions, originalTsConfig.compilerOptions || {}); -Object.assign(compilerOptions, { - moduleResolution: "node", - declaration: true, - rootDir: "." -}); -delete compilerOptions.allowJs; -fs.writeFileSync( - path.join(destinationDir, "tsconfig.json"), - JSON.stringify( - { - compilerOptions, - files: srcs.filter(src => src.endsWith(".ts") || src.endsWith(".tsx")) - }, - null, - 2 - ), - "utf8" -); From aceaae5ac3754efd4c92f6a64b4f70ca3b56b4cf Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 18 Oct 2018 15:11:48 -0700 Subject: [PATCH 09/43] Adding module collection --- examples/node-typescript-app/libs/shared-package/BUILD.bazel | 1 + internal/common/actions/create_source_dir/action.bzl | 5 ++++- .../common/actions/create_source_dir/create_source_dir.js | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/node-typescript-app/libs/shared-package/BUILD.bazel b/examples/node-typescript-app/libs/shared-package/BUILD.bazel index bf6d3ce..ff4ecf7 100644 --- a/examples/node-typescript-app/libs/shared-package/BUILD.bazel +++ b/examples/node-typescript-app/libs/shared-package/BUILD.bazel @@ -4,6 +4,7 @@ load("@bazel_javascript//:defs.bzl", "ts_library") ts_library( name = "shared-package", + module_name = "shared-package", srcs = glob(["**/*.ts"]), tsconfig = "//:tsconfig.json", deps = ["//:packages"], diff --git a/internal/common/actions/create_source_dir/action.bzl b/internal/common/actions/create_source_dir/action.bzl index a9d5ede..d0afaed 100644 --- a/internal/common/actions/create_source_dir/action.bzl +++ b/internal/common/actions/create_source_dir/action.bzl @@ -1,3 +1,6 @@ +def _map_modules(js_module_info): + return "%s:%s".format(js_module_info.module_root, js_module_info.module_name) + def create_source_dir(js, js_source, create_dir): """Creates a directory with the sources described in the JsSource object @@ -37,7 +40,7 @@ def create_source_dir(js, js_source, create_dir): script_args.add_all(library.all_sources, format_each = "s:%s") script_args.add_all(library.module_paths, format_each = "mrs:%s/node_modules/") - script_args.add_all(library.all_modules , format_each = "m:%s") + script_args.add_all(library.all_modules , map_each = _map_modules, format_each = "m:%s") inputs = depset( direct = direct_inputs, diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index ae609e8..c65f2eb 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -38,6 +38,7 @@ BazelAction( async args => { const { current_target, workspace_name, package_path, from, into } = args; const sources = ensureArray(args._); + console.log(sources); const nodeModulesPath = path.join(into, "node_modules"); const package = { workspace: workspace_name, @@ -79,6 +80,10 @@ BazelAction( ); } } + } else { + const fromModuleDir = parsed.path; + const moduleName = path.basename(parsed.path); + makeSymlink(fromModuleDir, path.join(nodeModulesPath, moduleName)); } } else if (parsed.symlink) { makeSymlink(parsed.path, path.join(into, parsed.path)); From 4ac8ccc5315529a05cb844a7361cb19ae2c524ff Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Mon, 22 Oct 2018 21:20:33 -0700 Subject: [PATCH 10/43] Format with buildifier --- internal/BUILD.bazel | 6 +++--- internal/common/BUILD.bazel | 2 +- internal/common/actions/BUILD.bazel | 2 +- .../common/actions/create_source_dir/BUILD.bazel | 2 +- internal/common/actions/create_source_dir/action.bzl | 6 +++--- internal/common/actions/run_js/BUILD.bazel | 2 +- internal/js_binary/BUILD.bazel | 4 ++-- internal/js_library/BUILD.bazel | 6 +++--- internal/js_module/BUILD.bazel | 2 +- internal/js_script_and_test/BUILD.bazel | 4 ++-- internal/npm_binary/BUILD.bazel | 2 +- internal/npm_packages/BUILD.bazel | 4 ++-- internal/npm_packages/empty/BUILD.bazel | 6 +++--- internal/ts_library/BUILD.bazel | 12 ++++++------ internal/web_bundle/BUILD.bazel | 6 +++--- 15 files changed, 33 insertions(+), 33 deletions(-) diff --git a/internal/BUILD.bazel b/internal/BUILD.bazel index be17003..bce860f 100644 --- a/internal/BUILD.bazel +++ b/internal/BUILD.bazel @@ -3,7 +3,7 @@ package(default_visibility = ["//visibility:public"]) load("//:defs.bzl", "npm_packages") npm_packages( - name = "packages", - package_json = ":package.json", - yarn_lock = ":yarn.lock", + name = "packages", + package_json = ":package.json", + yarn_lock = ":yarn.lock", ) diff --git a/internal/common/BUILD.bazel b/internal/common/BUILD.bazel index 15afdb9..11cd70f 100644 --- a/internal/common/BUILD.bazel +++ b/internal/common/BUILD.bazel @@ -1,5 +1,5 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "context.bzl" + "context.bzl", ]) diff --git a/internal/common/actions/BUILD.bazel b/internal/common/actions/BUILD.bazel index ac8ec7b..913e4d5 100644 --- a/internal/common/actions/BUILD.bazel +++ b/internal/common/actions/BUILD.bazel @@ -1,3 +1,3 @@ package(default_visibility = ["//visibility:public"]) -export_files(["actions.bzl"]) \ No newline at end of file +export_files(["actions.bzl"]) diff --git a/internal/common/actions/create_source_dir/BUILD.bazel b/internal/common/actions/create_source_dir/BUILD.bazel index 0fd2399..6acc720 100644 --- a/internal/common/actions/create_source_dir/BUILD.bazel +++ b/internal/common/actions/create_source_dir/BUILD.bazel @@ -3,4 +3,4 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "action.bzl", "create_source_dir.js", -]) \ No newline at end of file +]) diff --git a/internal/common/actions/create_source_dir/action.bzl b/internal/common/actions/create_source_dir/action.bzl index d0afaed..d0f330c 100644 --- a/internal/common/actions/create_source_dir/action.bzl +++ b/internal/common/actions/create_source_dir/action.bzl @@ -1,5 +1,5 @@ def _map_modules(js_module_info): - return "%s:%s".format(js_module_info.module_root, js_module_info.module_name) + return "%s:%s" % (js_module_info.module_root.path, js_module_info.module_name) def create_source_dir(js, js_source, create_dir): """Creates a directory with the sources described in the JsSource object @@ -44,7 +44,7 @@ def create_source_dir(js, js_source, create_dir): inputs = depset( direct = direct_inputs, - transitive = transitive_inputs, + transitive = [library.all_sources, library.all_modules], ) - js.run_js(js, inputs = inputs, outputs = [create_dir], script = js._create_source_dir_js, script_args = script_args) \ No newline at end of file + js.run_js(js, inputs = inputs, outputs = [create_dir], script = js._create_source_dir_js, script_args = script_args) diff --git a/internal/common/actions/run_js/BUILD.bazel b/internal/common/actions/run_js/BUILD.bazel index 17669fd..6f1734e 100644 --- a/internal/common/actions/run_js/BUILD.bazel +++ b/internal/common/actions/run_js/BUILD.bazel @@ -3,4 +3,4 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "action.bzl", "BazelAction.js", -]) \ No newline at end of file +]) diff --git a/internal/js_binary/BUILD.bazel b/internal/js_binary/BUILD.bazel index 9acde6a..1cc6ecf 100644 --- a/internal/js_binary/BUILD.bazel +++ b/internal/js_binary/BUILD.bazel @@ -1,6 +1,6 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "compile.js", - "rule.bzl", + "compile.js", + "rule.bzl", ]) diff --git a/internal/js_library/BUILD.bazel b/internal/js_library/BUILD.bazel index ec87f9d..ff4efb4 100644 --- a/internal/js_library/BUILD.bazel +++ b/internal/js_library/BUILD.bazel @@ -1,7 +1,7 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "compile.js", - "create_full_src.js", - "rule.bzl", + "compile.js", + "create_full_src.js", + "rule.bzl", ]) diff --git a/internal/js_module/BUILD.bazel b/internal/js_module/BUILD.bazel index 24e6e2f..368a810 100644 --- a/internal/js_module/BUILD.bazel +++ b/internal/js_module/BUILD.bazel @@ -1,5 +1,5 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "rule.bzl", + "rule.bzl", ]) diff --git a/internal/js_script_and_test/BUILD.bazel b/internal/js_script_and_test/BUILD.bazel index 9acde6a..1cc6ecf 100644 --- a/internal/js_script_and_test/BUILD.bazel +++ b/internal/js_script_and_test/BUILD.bazel @@ -1,6 +1,6 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "compile.js", - "rule.bzl", + "compile.js", + "rule.bzl", ]) diff --git a/internal/npm_binary/BUILD.bazel b/internal/npm_binary/BUILD.bazel index 24e6e2f..368a810 100644 --- a/internal/npm_binary/BUILD.bazel +++ b/internal/npm_binary/BUILD.bazel @@ -1,5 +1,5 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "rule.bzl", + "rule.bzl", ]) diff --git a/internal/npm_packages/BUILD.bazel b/internal/npm_packages/BUILD.bazel index 210e8f6..f665422 100644 --- a/internal/npm_packages/BUILD.bazel +++ b/internal/npm_packages/BUILD.bazel @@ -1,6 +1,6 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "install.js", - "rule.bzl", + "install.js", + "rule.bzl", ]) diff --git a/internal/npm_packages/empty/BUILD.bazel b/internal/npm_packages/empty/BUILD.bazel index 6403e06..37cc29b 100644 --- a/internal/npm_packages/empty/BUILD.bazel +++ b/internal/npm_packages/empty/BUILD.bazel @@ -5,7 +5,7 @@ load("//:defs.bzl", "npm_packages") # This rule is used by ts_library targets that don't depend on any npm_packages # targets (because they don't use any external NPM packages). npm_packages( - name = "packages", - package_json = ":package.json", - yarn_lock = ":yarn.lock", + name = "packages", + package_json = ":package.json", + yarn_lock = ":yarn.lock", ) diff --git a/internal/ts_library/BUILD.bazel b/internal/ts_library/BUILD.bazel index 34c8465..8d87b78 100644 --- a/internal/ts_library/BUILD.bazel +++ b/internal/ts_library/BUILD.bazel @@ -1,10 +1,10 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "compile.js", - "create_full_src.js", - "default_tsconfig.json", - "rule.bzl", - "transpile.js", - "tsconfig.gen.js", + "compile.js", + "create_full_src.js", + "default_tsconfig.json", + "rule.bzl", + "transpile.js", + "tsconfig.gen.js", ]) diff --git a/internal/web_bundle/BUILD.bazel b/internal/web_bundle/BUILD.bazel index 4ca8da3..1ac7d24 100644 --- a/internal/web_bundle/BUILD.bazel +++ b/internal/web_bundle/BUILD.bazel @@ -1,7 +1,7 @@ package(default_visibility = ["//visibility:public"]) exports_files([ - "compile.js", - "create_webpack_config.js", - "rule.bzl", + "compile.js", + "create_webpack_config.js", + "rule.bzl", ]) From d1cb028ac82c0e72395b09021b8f478518ea4f92 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 10:35:06 -0700 Subject: [PATCH 11/43] Adding module target depsets for action inputs --- .../actions/create_source_dir/action.bzl | 12 +-- internal/common/context.bzl | 98 ++++++++++++------- 2 files changed, 68 insertions(+), 42 deletions(-) diff --git a/internal/common/actions/create_source_dir/action.bzl b/internal/common/actions/create_source_dir/action.bzl index d0f330c..a235800 100644 --- a/internal/common/actions/create_source_dir/action.bzl +++ b/internal/common/actions/create_source_dir/action.bzl @@ -6,8 +6,8 @@ def create_source_dir(js, js_source, create_dir): Args: js: JsContext object - js_source_provider: JsSource object describing the sources to symlink in the directory - source_dir: File object to populate with the sources (eg. ctx.outputs.compilation_src_dir) + js_source: JsSource object describing the sources to symlink in the directory + create_dir: File object to populate with the sources (eg. ctx.outputs.compilation_src_dir) Returns: Array with JsSource provider in it @@ -19,7 +19,7 @@ def create_source_dir(js, js_source, create_dir): gen_scripts += js_source.gen_scripts direct_inputs = [] - direct_inputs += library.module_paths + direct_inputs += library.node_modules_dirs transitive_inputs = [library.all_sources] # Depset with all of the sources in it @@ -39,12 +39,12 @@ def create_source_dir(js, js_source, create_dir): script_args.add(argValue) script_args.add_all(library.all_sources, format_each = "s:%s") - script_args.add_all(library.module_paths, format_each = "mrs:%s/node_modules/") - script_args.add_all(library.all_modules , map_each = _map_modules, format_each = "m:%s") + script_args.add_all(library.node_modules_dirs, format_each = "mrs:%s/node_modules/") + script_args.add_all(library.all_dep_modules, map_each = _map_modules, format_each = "m:%s") inputs = depset( direct = direct_inputs, - transitive = [library.all_sources, library.all_modules], + transitive = [library.all_sources, library.all_dep_module_targets], ) js.run_js(js, inputs = inputs, outputs = [create_dir], script = js._create_source_dir_js, script_args = script_args) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 8ebd79f..4112129 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -32,11 +32,15 @@ JsLibraryInfo = provider( # All source files provided as input "all_sources", # Entire paths to add to npm modules (eg. a node_modules path) - "module_paths", + "node_modules_dirs", # Modules that are depended upon directly - "modules", + "dep_modules", # Transitive module dependencies - "all_modules", + "all_dep_modules", + # The target directories for the dependent modules + "dep_module_targets", + # The target directories for the transitive and directly dependent modules + "all_dep_module_targets", ], ) """Metadata about an individual js library. @@ -48,7 +52,7 @@ as: module dependencies, source files, paths to merge into node_modules. JsSourceInfo = provider(fields = [ # The source js files "srcs", - # List of scripts to output + # List of scripts to output "gen_scripts", # The library rule that generated this source "library", @@ -60,12 +64,18 @@ JsModuleInfo = provider( fields = [ # The root of the workspace "workspace_name", - # The root of the module + # The root directory of the module "module_root", # Name that will be used for non-relative imports "module_name", - # Transitive dependencies for this module - "all_modules", + # Modules that this module directly depends on + "dep_modules", + # All modules that this module and its dependencies require + "all_dep_modules", + # The target directories for the dependent modules + "dep_module_targets", + # The target directories for the transitive and directly dependent modules + "all_dep_module_targets", ], ) """ Wraps a JsLibrary with a package_name for nonrelative imports @@ -76,8 +86,8 @@ nonrelative imports JsModuleMap = provider( fields = [ - "module_map" - ] + "module_map", + ], ) ############################################################################### @@ -154,46 +164,61 @@ def _js_library_info(js, attr = None): deps_attr = getattr(attr, "deps", []) transitive_sources = [] - direct_modules = [] - transitive_modules = [] - module_paths = [] + node_modules_dirs = [] + dep_modules = [] + transitive_dep_modules = [] + dep_module_targets = [] + transitive_dep_module_targets = [] # Iterate through the deps to add them to their correct JsLibrary attributes for dep in deps_attr: - if JsLibraryInfo in dep: - dep_js_library = dep[JsLibraryInfo] - # The dependency is another JsLibrary - transitive_sources.append(dep_js_library.all_sources) - transitive_modules.append(dep_js_library.all_modules) - elif hasattr(dep, "tags") and "NODE_MODULES_MARKER" in getattr(dep, "tags"): - # The dependency is a module defined by rules_nodejs - direct_modules += dep - elif NpmPackagesInfo in dep: - # The dependency is a node_modules directory installed by npm_packages - module_paths.append(dep[NpmPackagesInfo].installed_dir) - - if JsModuleInfo in dep: - dep_js_module = dep[JsModuleInfo] - direct_modules.append(dep_js_module) - transitive_modules.append(dep_js_module.all_modules) - + if JsLibraryInfo in dep: + dep_js_library = dep[JsLibraryInfo] + + # The dependency is another JsLibrary + transitive_sources.append(dep_js_library.all_sources) + transitive_dep_modules.append(dep_js_library.all_dep_modules) + elif JsModuleInfo in dep: + dep_js_module = dep[JsModuleInfo] + transitive_dep_modules.append(dep_js_module.all_dep_modules) + transitive_dep_module_targets.append(dep_js_module.all_dep_module_targets) + elif hasattr(dep, "tags") and "NODE_MODULES_MARKER" in getattr(dep, "tags"): + # The dependency is a module defined by rules_nodejs + direct_modules += dep + + if JsModuleInfo in dep: + dep_js_module = dep[JsModuleInfo] + dep_module_targets.append(dep_js_module.module_root) + dep_modules.append(dep_js_module) + + if NpmPackagesInfo in dep: + # The dependency is a node_modules directory installed by npm_packages + node_modules_dirs.append(dep[NpmPackagesInfo].installed_dir) + all_src_files = depset( direct = src_files, transitive = transitive_sources, ) - all_modules = depset( - direct = direct_modules, - transitive = transitive_modules, + all_dep_modules = depset( + direct = dep_modules, + transitive = transitive_dep_modules, + ) + + all_dep_module_targets = depset( + direct = dep_module_targets, + transitive = transitive_dep_module_targets, ) return JsLibraryInfo( package_path = js.package_path, srcs = src_files, all_sources = all_src_files, - module_paths = module_paths, - modules = direct_modules, - all_modules = all_modules, + node_modules_dirs = node_modules_dirs, + dep_modules = dep_modules, + all_dep_modules = all_dep_modules, + dep_module_targets = dep_module_targets, + all_dep_module_targets = all_dep_module_targets, ) def _library_to_source_info(js, library, gen_scripts = None): @@ -211,7 +236,8 @@ def _library_to_source_info(js, library, gen_scripts = None): def _library_to_module_info(js, library, module_root, module_name): return JsModuleInfo( workspace_name = js.workspace_name, - all_modules = library.all_modules, + all_dep_modules = library.all_dep_modules, + all_dep_module_targets = library.all_dep_module_targets, module_root = module_root, module_name = module_name, ) From 4c3a904df34e5ec6a36ef30f12e85d49991d384c Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 10:43:35 -0700 Subject: [PATCH 12/43] Use module name from params --- .../common/actions/create_source_dir/create_source_dir.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index c65f2eb..083ee55 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -82,7 +82,9 @@ BazelAction( } } else { const fromModuleDir = parsed.path; - const moduleName = path.basename(parsed.path); + const moduleName = parsed.params[0] + ? parsed.params[0] + : path.basename(parsed.path); makeSymlink(fromModuleDir, path.join(nodeModulesPath, moduleName)); } } else if (parsed.symlink) { From f6f4c31d8a4038cca9269c350b38493024ad30f2 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 10:45:53 -0700 Subject: [PATCH 13/43] Removing verbose logging --- internal/common/actions/create_source_dir/create_source_dir.js | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index 083ee55..8ee629c 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -50,7 +50,6 @@ BazelAction( const populateFiles = async source => { const parsed = parseSource(source); - console.dir(parsed); /** * Node Module Population */ From b79aa66037856d0c7ed88ea3e814607aa7b479e1 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 11:01:20 -0700 Subject: [PATCH 14/43] Adding the original js ctx to all providers --- internal/common/context.bzl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 4112129..b04b195 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -23,6 +23,7 @@ JsContext = provider() JsLibraryInfo = provider( fields = [ + "js", # The label that produced the JsLibrary "label", # Path of the BUILD.bazel file relative to the workspace root. @@ -50,6 +51,7 @@ as: module dependencies, source files, paths to merge into node_modules. """ JsSourceInfo = provider(fields = [ + "js", # The source js files "srcs", # List of scripts to output @@ -62,6 +64,7 @@ JsSourceInfo = provider(fields = [ JsModuleInfo = provider( fields = [ + "js", # The root of the workspace "workspace_name", # The root directory of the module @@ -211,6 +214,7 @@ def _js_library_info(js, attr = None): ) return JsLibraryInfo( + js = js, package_path = js.package_path, srcs = src_files, all_sources = all_src_files, @@ -228,6 +232,7 @@ def _library_to_source_info(js, library, gen_scripts = None): """ return JsSourceInfo( + js = js, srcs = library.all_sources, gen_scripts = gen_scripts, library = library, @@ -235,6 +240,7 @@ def _library_to_source_info(js, library, gen_scripts = None): def _library_to_module_info(js, library, module_root, module_name): return JsModuleInfo( + js = js, workspace_name = js.workspace_name, all_dep_modules = library.all_dep_modules, all_dep_module_targets = library.all_dep_module_targets, From 75f677332c54ab2aae63c4a34edcd5fcd80a5560 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 11:01:36 -0700 Subject: [PATCH 15/43] Resolving modules all the way to their package directory --- .../actions/create_source_dir/action.bzl | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/common/actions/create_source_dir/action.bzl b/internal/common/actions/create_source_dir/action.bzl index a235800..222f908 100644 --- a/internal/common/actions/create_source_dir/action.bzl +++ b/internal/common/actions/create_source_dir/action.bzl @@ -1,5 +1,5 @@ def _map_modules(js_module_info): - return "%s:%s" % (js_module_info.module_root.path, js_module_info.module_name) + return "%s/%s:%s" % (js_module_info.module_root.path, js_module_info.js.package_path, js_module_info.module_name) def create_source_dir(js, js_source, create_dir): """Creates a directory with the sources described in the JsSource object @@ -14,9 +14,9 @@ def create_source_dir(js, js_source, create_dir): """ library = js_source.library - gen_scripts = [] + gen_scripts = [] if js_source.gen_scripts: - gen_scripts += js_source.gen_scripts + gen_scripts += js_source.gen_scripts direct_inputs = [] direct_inputs += library.node_modules_dirs @@ -28,23 +28,23 @@ def create_source_dir(js, js_source, create_dir): script_args.add("--from", js.package_path) for gen_script in gen_scripts: - if type(gen_script) == type(""): - direct_inputs.append(gen_script) - script_args.add("g:./{}".format(gen_script.path)) - elif type(gen_script) == type([]) and len(gen_script) > 0: - argValue = "g" - for value in gen_script: - direct_inputs.append(value) - argValue += ":./{}".format(value.path) - script_args.add(argValue) + if type(gen_script) == type(""): + direct_inputs.append(gen_script) + script_args.add("g:./{}".format(gen_script.path)) + elif type(gen_script) == type([]) and len(gen_script) > 0: + argValue = "g" + for value in gen_script: + direct_inputs.append(value) + argValue += ":./{}".format(value.path) + script_args.add(argValue) script_args.add_all(library.all_sources, format_each = "s:%s") script_args.add_all(library.node_modules_dirs, format_each = "mrs:%s/node_modules/") script_args.add_all(library.all_dep_modules, map_each = _map_modules, format_each = "m:%s") inputs = depset( - direct = direct_inputs, - transitive = [library.all_sources, library.all_dep_module_targets], + direct = direct_inputs, + transitive = [library.all_sources, library.all_dep_module_targets], ) js.run_js(js, inputs = inputs, outputs = [create_dir], script = js._create_source_dir_js, script_args = script_args) From 76fce50941a03899465c625c2c867afb9099c022 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 11:03:17 -0700 Subject: [PATCH 16/43] Commit formatted example files --- examples/node-typescript-app/BUILD.bazel | 6 +- .../libs/shared-package/BUILD.bazel | 10 ++-- .../services/base-image/BUILD.bazel | 8 +-- .../services/my-service/BUILD.bazel | 60 +++++++++---------- .../services/my-service/server/BUILD.bazel | 14 ++--- examples/react-app-javascript/BUILD.bazel | 26 ++++---- .../react-app-javascript/public/BUILD.bazel | 2 +- examples/react-app-javascript/src/BUILD.bazel | 20 +++---- examples/react-app-typescript/BUILD.bazel | 26 ++++---- .../react-app-typescript/public/BUILD.bazel | 2 +- examples/react-app-typescript/src/BUILD.bazel | 22 +++---- .../react-storybook-typescript/BUILD.bazel | 14 ++--- .../src/BUILD.bazel | 52 ++++++++-------- 13 files changed, 131 insertions(+), 131 deletions(-) diff --git a/examples/node-typescript-app/BUILD.bazel b/examples/node-typescript-app/BUILD.bazel index 86be255..1961b59 100644 --- a/examples/node-typescript-app/BUILD.bazel +++ b/examples/node-typescript-app/BUILD.bazel @@ -5,7 +5,7 @@ exports_files(["tsconfig.json"]) load("@bazel_javascript//:defs.bzl", "npm_packages") npm_packages( - name = "packages", - package_json = ":package.json", - yarn_lock = ":yarn.lock", + name = "packages", + package_json = ":package.json", + yarn_lock = ":yarn.lock", ) diff --git a/examples/node-typescript-app/libs/shared-package/BUILD.bazel b/examples/node-typescript-app/libs/shared-package/BUILD.bazel index ff4ecf7..89633ae 100644 --- a/examples/node-typescript-app/libs/shared-package/BUILD.bazel +++ b/examples/node-typescript-app/libs/shared-package/BUILD.bazel @@ -3,9 +3,9 @@ package(default_visibility = ["//visibility:public"]) load("@bazel_javascript//:defs.bzl", "ts_library") ts_library( - name = "shared-package", - module_name = "shared-package", - srcs = glob(["**/*.ts"]), - tsconfig = "//:tsconfig.json", - deps = ["//:packages"], + name = "shared-package", + srcs = glob(["**/*.ts"]), + module_name = "shared-package", + tsconfig = "//:tsconfig.json", + deps = ["//:packages"], ) diff --git a/examples/node-typescript-app/services/base-image/BUILD.bazel b/examples/node-typescript-app/services/base-image/BUILD.bazel index 5a667b5..1eb0b05 100644 --- a/examples/node-typescript-app/services/base-image/BUILD.bazel +++ b/examples/node-typescript-app/services/base-image/BUILD.bazel @@ -4,8 +4,8 @@ load("@io_bazel_rules_docker//container:container.bzl", "container_image", "cont ## The Backend Base Image container_image( - name = "service", - base = "@node_alpine_image//image:image", - stamp = True, - workdir = "/app", + name = "service", + base = "@node_alpine_image//image:image", + stamp = True, + workdir = "/app", ) diff --git a/examples/node-typescript-app/services/my-service/BUILD.bazel b/examples/node-typescript-app/services/my-service/BUILD.bazel index fc7c44b..708949b 100644 --- a/examples/node-typescript-app/services/my-service/BUILD.bazel +++ b/examples/node-typescript-app/services/my-service/BUILD.bazel @@ -3,37 +3,37 @@ package(default_visibility = ["//visibility:public"]) load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_push") container_image( - name = "image", - base = "//services/base-image:service", - data_path = ".", - directory = "/app", - files = [ - "//:packages", - "//services/my-service/server:server_compiled", - ], - symlinks = { - "/app/node_modules": "/app/packages_installed_dir/node_modules", - }, - cmd = [ - "sh", - "-c", - # Rebuild packages such as node-sass, since they may not have been downloaded by Bazel with the right architecture. - # Ideally, this should be done when the image is built, not when it's run. Does rules_docker allow this? - "npm rebuild && " + - "node -r source-map-support/register server/server_compiled/services/my-service/server/server.js", - ], - ports = [ - "3000", - ], - stamp = True, + name = "image", + base = "//services/base-image:service", + cmd = [ + "sh", + "-c", + # Rebuild packages such as node-sass, since they may not have been downloaded by Bazel with the right architecture. + # Ideally, this should be done when the image is built, not when it's run. Does rules_docker allow this? + "npm rebuild && " + + "node -r source-map-support/register server/server_compiled/services/my-service/server/server.js", + ], + data_path = ".", + directory = "/app", + files = [ + "//:packages", + "//services/my-service/server:server_compiled", + ], + ports = [ + "3000", + ], + stamp = True, + symlinks = { + "/app/node_modules": "/app/packages_installed_dir/node_modules", + }, ) container_push( - name = "publish", - format = "Docker", - image = ":image", - registry = "my.repo.com", - repository = "my-service", - stamp = True, - tag = "{BUILD_USER}", + name = "publish", + format = "Docker", + image = ":image", + registry = "my.repo.com", + repository = "my-service", + stamp = True, + tag = "{BUILD_USER}", ) diff --git a/examples/node-typescript-app/services/my-service/server/BUILD.bazel b/examples/node-typescript-app/services/my-service/server/BUILD.bazel index d8b69d8..0632330 100644 --- a/examples/node-typescript-app/services/my-service/server/BUILD.bazel +++ b/examples/node-typescript-app/services/my-service/server/BUILD.bazel @@ -3,11 +3,11 @@ package(default_visibility = ["//visibility:public"]) load("@bazel_javascript//:defs.bzl", "ts_library") ts_library( - name = "server", - srcs = glob(["**/*.ts"]), - tsconfig = "//:tsconfig.json", - deps = [ - "//:packages", - "//libs/shared-package", - ], + name = "server", + srcs = glob(["**/*.ts"]), + tsconfig = "//:tsconfig.json", + deps = [ + "//:packages", + "//libs/shared-package", + ], ) diff --git a/examples/react-app-javascript/BUILD.bazel b/examples/react-app-javascript/BUILD.bazel index 4ebdb96..686fde1 100644 --- a/examples/react-app-javascript/BUILD.bazel +++ b/examples/react-app-javascript/BUILD.bazel @@ -3,23 +3,23 @@ package(default_visibility = ["//visibility:public"]) load("@bazel_javascript//:defs.bzl", "npm_packages", "web_bundle") web_bundle( - name = "app-bundle-dev", - lib = "//src", - entry = "index.js", - mode = "development", - html_template = "//public:index.html", + name = "app-bundle-dev", + entry = "index.js", + html_template = "//public:index.html", + lib = "//src", + mode = "development", ) web_bundle( - name = "app-bundle-prod", - lib = "//src", - entry = "index.js", - mode = "production", - html_template = "//public:index.html", + name = "app-bundle-prod", + entry = "index.js", + html_template = "//public:index.html", + lib = "//src", + mode = "production", ) npm_packages( - name = "packages", - package_json = ":package.json", - yarn_lock = ":yarn.lock", + name = "packages", + package_json = ":package.json", + yarn_lock = ":yarn.lock", ) diff --git a/examples/react-app-javascript/public/BUILD.bazel b/examples/react-app-javascript/public/BUILD.bazel index 7955fc1..264d8bd 100644 --- a/examples/react-app-javascript/public/BUILD.bazel +++ b/examples/react-app-javascript/public/BUILD.bazel @@ -1,3 +1,3 @@ exports_files([ - "index.html", + "index.html", ]) diff --git a/examples/react-app-javascript/src/BUILD.bazel b/examples/react-app-javascript/src/BUILD.bazel index fb2b90c..308620b 100644 --- a/examples/react-app-javascript/src/BUILD.bazel +++ b/examples/react-app-javascript/src/BUILD.bazel @@ -3,14 +3,14 @@ package(default_visibility = ["//visibility:public"]) load("@bazel_javascript//:defs.bzl", "js_library") js_library( - name = "src", - srcs = glob([ - "*.js", - "*.jsx", - "*.css", - "*.svg", - ]), - deps = [ - "//:packages", - ], + name = "src", + srcs = glob([ + "*.js", + "*.jsx", + "*.css", + "*.svg", + ]), + deps = [ + "//:packages", + ], ) diff --git a/examples/react-app-typescript/BUILD.bazel b/examples/react-app-typescript/BUILD.bazel index 11ef3a4..1432244 100644 --- a/examples/react-app-typescript/BUILD.bazel +++ b/examples/react-app-typescript/BUILD.bazel @@ -5,23 +5,23 @@ load("@bazel_javascript//:defs.bzl", "npm_packages", "web_bundle") exports_files(["tsconfig.json"]) web_bundle( - name = "app-bundle-dev", - lib = "//src", - entry = "index.js", - mode = "development", - html_template = "//public:index.html", + name = "app-bundle-dev", + entry = "index.js", + html_template = "//public:index.html", + lib = "//src", + mode = "development", ) web_bundle( - name = "app-bundle-prod", - lib = "//src", - entry = "index.js", - mode = "production", - html_template = "//public:index.html", + name = "app-bundle-prod", + entry = "index.js", + html_template = "//public:index.html", + lib = "//src", + mode = "production", ) npm_packages( - name = "packages", - package_json = ":package.json", - yarn_lock = ":yarn.lock", + name = "packages", + package_json = ":package.json", + yarn_lock = ":yarn.lock", ) diff --git a/examples/react-app-typescript/public/BUILD.bazel b/examples/react-app-typescript/public/BUILD.bazel index 7955fc1..264d8bd 100644 --- a/examples/react-app-typescript/public/BUILD.bazel +++ b/examples/react-app-typescript/public/BUILD.bazel @@ -1,3 +1,3 @@ exports_files([ - "index.html", + "index.html", ]) diff --git a/examples/react-app-typescript/src/BUILD.bazel b/examples/react-app-typescript/src/BUILD.bazel index 68c9222..a5d884b 100644 --- a/examples/react-app-typescript/src/BUILD.bazel +++ b/examples/react-app-typescript/src/BUILD.bazel @@ -3,15 +3,15 @@ package(default_visibility = ["//visibility:public"]) load("@bazel_javascript//:defs.bzl", "ts_library") ts_library( - name = "src", - srcs = glob([ - "*.ts", - "*.tsx", - "*.css", - "*.svg", - ]), - tsconfig = "//:tsconfig.json", - deps = [ - "//:packages", - ], + name = "src", + srcs = glob([ + "*.ts", + "*.tsx", + "*.css", + "*.svg", + ]), + tsconfig = "//:tsconfig.json", + deps = [ + "//:packages", + ], ) diff --git a/examples/react-storybook-typescript/BUILD.bazel b/examples/react-storybook-typescript/BUILD.bazel index 7723e8c..7e971bd 100644 --- a/examples/react-storybook-typescript/BUILD.bazel +++ b/examples/react-storybook-typescript/BUILD.bazel @@ -1,17 +1,17 @@ package(default_visibility = ["//visibility:public"]) -load("@bazel_javascript//:defs.bzl", "npm_packages", "js_script") +load("@bazel_javascript//:defs.bzl", "js_script", "npm_packages") exports_files(["tsconfig.json"]) js_script( - name = "storybook-run", - cmd = "start-storybook -p 9001 -c $LIB_DIR/.storybook", - lib = "//src:storybook", + name = "storybook-run", + cmd = "start-storybook -p 9001 -c $LIB_DIR/.storybook", + lib = "//src:storybook", ) npm_packages( - name = "packages", - package_json = ":package.json", - yarn_lock = ":yarn.lock", + name = "packages", + package_json = ":package.json", + yarn_lock = ":yarn.lock", ) diff --git a/examples/react-storybook-typescript/src/BUILD.bazel b/examples/react-storybook-typescript/src/BUILD.bazel index 43d314c..c3cd864 100644 --- a/examples/react-storybook-typescript/src/BUILD.bazel +++ b/examples/react-storybook-typescript/src/BUILD.bazel @@ -3,36 +3,36 @@ package(default_visibility = ["//visibility:public"]) load("@bazel_javascript//:defs.bzl", "ts_library") ts_library( - name = "storybook", - srcs = [ - ".storybook/config.ts", - ], - deps = [ - ":component_story", - "//:packages", - ], + name = "storybook", + srcs = [ + ".storybook/config.ts", + ], + deps = [ + ":component_story", + "//:packages", + ], ) ts_library( - name = "component_story", - srcs = [ - "component.story.tsx", - ], - deps = [ - ":component", - "//:packages", - ], + name = "component_story", + srcs = [ + "component.story.tsx", + ], + deps = [ + ":component", + "//:packages", + ], ) ts_library( - name = "component", - srcs = [ - "component1.tsx", - "component1.css", - "component2.tsx", - "component2.css", - ], - deps = [ - "//:packages", - ], + name = "component", + srcs = [ + "component1.css", + "component1.tsx", + "component2.css", + "component2.tsx", + ], + deps = [ + "//:packages", + ], ) From 9976189f41d93478104c6ecd5fe1bec64bf3b4a2 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 11:14:43 -0700 Subject: [PATCH 17/43] Removing verbose log --- internal/common/actions/create_source_dir/create_source_dir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index 8ee629c..772a050 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -38,7 +38,7 @@ BazelAction( async args => { const { current_target, workspace_name, package_path, from, into } = args; const sources = ensureArray(args._); - console.log(sources); + // console.log(sources); const nodeModulesPath = path.join(into, "node_modules"); const package = { workspace: workspace_name, From 63afafe79d2585b5495fd1122f318cb2c6b53e95 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Tue, 23 Oct 2018 16:19:03 -0700 Subject: [PATCH 18/43] Fix example --- internal/common/actions/create_source_dir/create_source_dir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index 772a050..9b1b4dc 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -24,7 +24,7 @@ const { BazelAction, ensureArray } = require("../run_js/BazelAction"); * mr: Recursive module (add all the modules to node_modules) * g: Generate (run the passed in script to generate source files * - * eg. node create_source_dir.js -s file/to/symlink. + * eg. node create_source_dir.js s:file/to/symlink. */ BazelAction( { From 16358573cacd6054560fedfbe43e79c06e4ee32a Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 12:10:52 -0700 Subject: [PATCH 19/43] Fixing exports_files --- internal/common/actions/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/actions/BUILD.bazel b/internal/common/actions/BUILD.bazel index 913e4d5..28418bc 100644 --- a/internal/common/actions/BUILD.bazel +++ b/internal/common/actions/BUILD.bazel @@ -1,3 +1,3 @@ package(default_visibility = ["//visibility:public"]) -export_files(["actions.bzl"]) +exports_files(["actions.bzl"]) From e4319ffe1f0b92987c0f0ab58e1e6988aa3fbf1b Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 13:53:28 -0700 Subject: [PATCH 20/43] Fixing comments per PR review --- .../actions/create_source_dir/create_source_dir.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index 9b1b4dc..054fba6 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -21,7 +21,8 @@ const { BazelAction, ensureArray } = require("../run_js/BazelAction"); * r: Recurse * rs:./some/dir/ * m: Module (put it in node_modules) - * mr: Recursive module (add all the modules to node_modules) + * mrs: Recursive module (add all the modules to node_modules) + * mrs:some/node_modules * g: Generate (run the passed in script to generate source files * * eg. node create_source_dir.js s:file/to/symlink. @@ -38,7 +39,6 @@ BazelAction( async args => { const { current_target, workspace_name, package_path, from, into } = args; const sources = ensureArray(args._); - // console.log(sources); const nodeModulesPath = path.join(into, "node_modules"); const package = { workspace: workspace_name, @@ -93,13 +93,6 @@ BazelAction( } }; - // console.log(`Current Target: ${current_target}`); - // console.log(`Target Source Path: ${from}`); - // console.log(`Copying into: ${into}`); - // console.log(`Running in [${process.cwd()}]`); - // console.log(`Environment is: ${JSON.stringify(process.env, null, 2)}`); - // console.log(`Will create [${into}] for populating`); - for (const source of sources) { await populateFiles(source); } From 083104f4fcfe0cedce6e41c2080e2b85884dafe7 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 13:54:56 -0700 Subject: [PATCH 21/43] Fixing documentation per PR comments --- internal/common/actions/create_source_dir/create_source_dir.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index 054fba6..a2a0dc0 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -23,7 +23,8 @@ const { BazelAction, ensureArray } = require("../run_js/BazelAction"); * m: Module (put it in node_modules) * mrs: Recursive module (add all the modules to node_modules) * mrs:some/node_modules - * g: Generate (run the passed in script to generate source files + * g: Generate (run the passed in script to generate source files) + * g:some/script.js * * eg. node create_source_dir.js s:file/to/symlink. */ From d37eede14454523969584fc1c066e0fd4f56a175 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 14:30:45 -0700 Subject: [PATCH 22/43] Removing invalid documentation Still might be a good idea to implement this using URIs instead though. --- .../common/actions/create_source_dir/create_source_dir.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index a2a0dc0..2da7c68 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -6,11 +6,6 @@ const { BazelAction, ensureArray } = require("../run_js/BazelAction"); * Action for creating a directory with the passed in files for symlinking and * copying. * - * The files can follow a uri syntax with params: - * recurse=false (default) - * symlink=true (default) - * - * * The files to be populated have flags set on them for the appropriate actions: * [flags]:./file/path * Possible flags: From 4772197db8fb864d25cb821ccb035f3436cc52ea Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 16:12:14 -0700 Subject: [PATCH 23/43] Removign extra comments per PR review --- internal/common/context.bzl | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index b04b195..7c5aab7 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -1,7 +1,5 @@ load("//internal/common/actions:actions.bzl", "create_source_dir", "run_js") load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") -# load("@rules_nodejs//internal/common:node_module_info.bzl", "NodeModuleInfo", "collect_node_modules_aspect") -# load("@rules_nodejs//internal/common:module_mappings.bzl", "get_module_mappings", "module_mappings_runtime_aspect") ############################################################################### # Providers @@ -9,15 +7,6 @@ load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") # The Providers used are inspired by the rules_go providers: # https://github.com/bazelbuild/rules_go/blob/master/go/providers.rst#GoLibrary -# The NodeModuleInfo is provided by the rules_nodejs and generated by the -# collect_node_modules_aspect when the target has a "tags" attribute with -# "NODE_MODULE_MARKER" in it. -# NodeModuleInfo = provider( -# fields = { -# Only the workspace is needed because rules_ -# "workspace": "The workspace name that the npm dependencies are provided from" -# }) - # Wrapper for rule ctx that should be created through js_context JsContext = provider() @@ -315,12 +304,12 @@ def js_context(ctx, attr = None): _actions_bazel_action_js = _actions_bazel_action_js, _create_source_dir_js = _create_source_dir_js, - #Actions + # Actions actions = ctx.actions, create_source_dir = create_source_dir, run_js = run_js, - #Helpers + # Helpers script_args = _script_args, library_info = _js_library_info, library_to_source_info = _library_to_source_info, From 8c523eba13e9b0b5c0b4d34021b477f9a53a9824 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 16:13:06 -0700 Subject: [PATCH 24/43] Removing unused JsModuleMap --- internal/common/context.bzl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 7c5aab7..f895a0d 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -76,12 +76,6 @@ Including this as a dependency should add the "package_name" as a key for nonrelative imports """ -JsModuleMap = provider( - fields = [ - "module_map", - ], -) - ############################################################################### # Common Attributes From bd639f60a21a39c5327cdbec9cbf77aeea585a7d Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 16:15:06 -0700 Subject: [PATCH 25/43] Removing cruft per PR comments --- internal/ts_library/rule.bzl | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/ts_library/rule.bzl b/internal/ts_library/rule.bzl index a39427e..a3a8d58 100644 --- a/internal/ts_library/rule.bzl +++ b/internal/ts_library/rule.bzl @@ -14,9 +14,6 @@ TsLibraryInfo = provider(fields = [ def _ts_library_impl(ctx): js = js_context(ctx) providers = [] - # for i, d in enumerate(ctx.attr.deps): - # print(" {}. label = {}".format(i + 1, d.label)) - # print(" files = " + str([f.path for f in d.files.to_list()])) js_library = js.library_info(js) providers.append(js_library) From 91674e11fdb81bf5dd559a941306ab61847cb18f Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 16:18:55 -0700 Subject: [PATCH 26/43] Fixing prettier formatting --- internal/common/actions/run_js/BazelAction.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/common/actions/run_js/BazelAction.js b/internal/common/actions/run_js/BazelAction.js index 2841687..788b904 100644 --- a/internal/common/actions/run_js/BazelAction.js +++ b/internal/common/actions/run_js/BazelAction.js @@ -122,5 +122,5 @@ function safeSymlink(fromPath, toPath) { module.exports = { safeSymlink, ensureArray, - BazelAction, -} \ No newline at end of file + BazelAction +}; From c75c9d91df7de8232498cbc8653bda179368e24b Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 16:22:58 -0700 Subject: [PATCH 27/43] Buildifier formatting --- internal/common/actions/run_js/action.bzl | 2 +- internal/ts_library/rule.bzl | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/common/actions/run_js/action.bzl b/internal/common/actions/run_js/action.bzl index 9f57897..b18225e 100644 --- a/internal/common/actions/run_js/action.bzl +++ b/internal/common/actions/run_js/action.bzl @@ -40,4 +40,4 @@ def run_js( executable = js._internal_nodejs, arguments = [script.path, script_args], env = env, - ) \ No newline at end of file + ) diff --git a/internal/ts_library/rule.bzl b/internal/ts_library/rule.bzl index a3a8d58..a73d2bc 100644 --- a/internal/ts_library/rule.bzl +++ b/internal/ts_library/rule.bzl @@ -19,7 +19,7 @@ def _ts_library_impl(ctx): providers.append(js_library) js_source = js.library_to_source_info(js, js_library, gen_scripts = [ - [ctx.file._ts_config_genscript, ctx.file.tsconfig] + [ctx.file._ts_config_genscript, ctx.file.tsconfig], ]) providers.append(js_source) @@ -28,10 +28,10 @@ def _ts_library_impl(ctx): if js.module_name: js_module = js.library_to_module_info( js, - js_library, - module_name = js.module_name, + js_library, + module_name = js.module_name, module_root = ctx.outputs.compiled_dir, - ) + ) providers.append(js_module) compile_args = js.script_args(js, ctx.file._ts_library_compile_script.path) @@ -51,7 +51,7 @@ def _ts_library_impl(ctx): script_args = compile_args, ) - return providers + return providers def _compiled_extension(path): if path.endswith(".tsx"): @@ -88,7 +88,7 @@ ts_library = rule( allow_files = True, single_file = True, default = Label("//internal/ts_library:tsconfig.gen.js"), - ) + ), }), outputs = { "compilation_src_dir": "%{name}_compilation_src", From 69c8d71b87d45f125e069504226e9bc0e55cf129 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Wed, 24 Oct 2018 16:25:10 -0700 Subject: [PATCH 28/43] Removing unused _compiled_extension function --- internal/ts_library/rule.bzl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/ts_library/rule.bzl b/internal/ts_library/rule.bzl index a73d2bc..18aa6ba 100644 --- a/internal/ts_library/rule.bzl +++ b/internal/ts_library/rule.bzl @@ -53,14 +53,6 @@ def _ts_library_impl(ctx): return providers -def _compiled_extension(path): - if path.endswith(".tsx"): - return path[:-4] + ".js" - elif path.endswith(".ts"): - return path[:-3] + ".js" - else: - return path - ts_library = rule( implementation = _ts_library_impl, attrs = dict(JS_LIBRARY_ATTRIBUTES, **{ From 43e67be6802f1874ebf0db2f5a201fca4906f743 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 25 Oct 2018 14:01:35 -0700 Subject: [PATCH 29/43] Fixing documentation of nod_modules folder symlinking --- internal/common/actions/create_source_dir/create_source_dir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index 2da7c68..76243b0 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -16,7 +16,7 @@ const { BazelAction, ensureArray } = require("../run_js/BazelAction"); * r: Recurse * rs:./some/dir/ * m: Module (put it in node_modules) - * mrs: Recursive module (add all the modules to node_modules) + * mrs: Symlink all folders in directory into node_modules * mrs:some/node_modules * g: Generate (run the passed in script to generate source files) * g:some/script.js From 5c2910d3fe8c2b3c74ed9f8ae05d59a80f756725 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 25 Oct 2018 14:02:17 -0700 Subject: [PATCH 30/43] Adding statement about org folders --- internal/common/actions/create_source_dir/create_source_dir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/create_source_dir.js b/internal/common/actions/create_source_dir/create_source_dir.js index 76243b0..667f9ef 100644 --- a/internal/common/actions/create_source_dir/create_source_dir.js +++ b/internal/common/actions/create_source_dir/create_source_dir.js @@ -16,7 +16,7 @@ const { BazelAction, ensureArray } = require("../run_js/BazelAction"); * r: Recurse * rs:./some/dir/ * m: Module (put it in node_modules) - * mrs: Symlink all folders in directory into node_modules + * mrs: Symlink all folders in directory into node_modules, decending into folders that start with '@' * mrs:some/node_modules * g: Generate (run the passed in script to generate source files) * g:some/script.js From 92e49e3a8a9734c9082267df9b3902ee1eec3f19 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Thu, 25 Oct 2018 14:04:48 -0700 Subject: [PATCH 31/43] Removing script path from creation of script_args --- internal/common/context.bzl | 4 +--- internal/ts_library/rule.bzl | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index f895a0d..0dac843 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -231,7 +231,7 @@ def _library_to_module_info(js, library, module_root, module_name): module_name = module_name, ) -def _script_args(js, script_file): +def _script_args(js): """Create Args object that can be used with js.run_js() Args: @@ -247,8 +247,6 @@ def _script_args(js, script_file): args.use_param_file("--param=%s") args.set_param_file_format("multiline") - # args.add(script_file) - args.add("--current_target", js.label) args.add("--workspace_name", js.workspace_name) args.add("--package_path", js.package_path) diff --git a/internal/ts_library/rule.bzl b/internal/ts_library/rule.bzl index 18aa6ba..d25b021 100644 --- a/internal/ts_library/rule.bzl +++ b/internal/ts_library/rule.bzl @@ -34,7 +34,7 @@ def _ts_library_impl(ctx): ) providers.append(js_module) - compile_args = js.script_args(js, ctx.file._ts_library_compile_script.path) + compile_args = js.script_args(js) compile_args.add("--project", ctx.outputs.compilation_src_dir) compile_args.add("--outDir", ctx.outputs.compiled_dir) From 489277ed8ef213f874d4d70dc4a382d1de184dda Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:18:50 -0800 Subject: [PATCH 32/43] Adding filegroup to add all of the common js files to a rules dependencies --- internal/common/BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/common/BUILD.bazel b/internal/common/BUILD.bazel index 11cd70f..f3035f0 100644 --- a/internal/common/BUILD.bazel +++ b/internal/common/BUILD.bazel @@ -1,5 +1,11 @@ package(default_visibility = ["//visibility:public"]) +filegroup( + name = "_common_js_files", + srcs = glob(["**/*.js"]) +) + exports_files([ "context.bzl", + "index.js", ]) From 4df8c1c37b417c1aff55740d2fa8320ffddb8c61 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:19:49 -0800 Subject: [PATCH 33/43] using src_files and all_src_files to make clearer This makes it obvious that the file targets are coming from teh srcs attribute. --- internal/common/context.bzl | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 0dac843..4263ff4 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -18,9 +18,9 @@ JsLibraryInfo = provider( # Path of the BUILD.bazel file relative to the workspace root. "package_path", # Source files provided as input. - "srcs", + "src_files", # All source files provided as input - "all_sources", + "all_src_files", # Entire paths to add to npm modules (eg. a node_modules path) "node_modules_dirs", # Modules that are depended upon directly @@ -42,7 +42,7 @@ as: module dependencies, source files, paths to merge into node_modules. JsSourceInfo = provider(fields = [ "js", # The source js files - "srcs", + "src_files", # List of scripts to output "gen_scripts", # The library rule that generated this source @@ -81,6 +81,10 @@ nonrelative imports # Attributes that should be included for any rule that wants to create a JsContext JS_CONTEXT_ATTRIBUTES = { + "_common_js_files": attr.label( + allow_files = True, + default = Label("//internal/common:_common_js_files") + ) "_actions_bazel_action_js": attr.label( allow_files = True, single_file = True, @@ -199,8 +203,8 @@ def _js_library_info(js, attr = None): return JsLibraryInfo( js = js, package_path = js.package_path, - srcs = src_files, - all_sources = all_src_files, + src_files = src_files, + all_src_files = all_src_files, node_modules_dirs = node_modules_dirs, dep_modules = dep_modules, all_dep_modules = all_dep_modules, @@ -216,7 +220,7 @@ def _library_to_source_info(js, library, gen_scripts = None): return JsSourceInfo( js = js, - srcs = library.all_sources, + src_files = library.all_src_files, gen_scripts = gen_scripts, library = library, ) @@ -238,9 +242,6 @@ def _script_args(js): js: JsContext object script_file: File object for the script to be run """ - if script_file == None: - fail("No script file provided to script args") - args = js.actions.args() # If the args get too big then spill over into the param file From 7dfff1ddc33cd1549238c4000608fcce56f35de4 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:20:55 -0800 Subject: [PATCH 34/43] Removing unneeded script from the args --- internal/common/actions/create_source_dir/action.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/actions/create_source_dir/action.bzl b/internal/common/actions/create_source_dir/action.bzl index 222f908..d7b89fc 100644 --- a/internal/common/actions/create_source_dir/action.bzl +++ b/internal/common/actions/create_source_dir/action.bzl @@ -23,7 +23,7 @@ def create_source_dir(js, js_source, create_dir): transitive_inputs = [library.all_sources] # Depset with all of the sources in it - script_args = js.script_args(js, js._create_source_dir_js) + script_args = js.script_args(js) script_args.add("--into", create_dir) script_args.add("--from", js.package_path) From 7c29fbfca59f8a9864a847f77a12fdb036ce8094 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:22:40 -0800 Subject: [PATCH 35/43] Adding in index file for the common scripts --- internal/common/index.js | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 internal/common/index.js diff --git a/internal/common/index.js b/internal/common/index.js new file mode 100644 index 0000000..feb2076 --- /dev/null +++ b/internal/common/index.js @@ -0,0 +1,5 @@ +const BazelAction = require("./actions/run_js/BazelAction"); + +module.exports = { + ...BazelAction +}; From 57e276167686b655a49bc8dd982b1a663feb0721 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:23:09 -0800 Subject: [PATCH 36/43] Removing unneeded script --- internal/js_library/create_full_src.js | 47 -------------------------- 1 file changed, 47 deletions(-) delete mode 100644 internal/js_library/create_full_src.js diff --git a/internal/js_library/create_full_src.js b/internal/js_library/create_full_src.js deleted file mode 100644 index ecd6e79..0000000 --- a/internal/js_library/create_full_src.js +++ /dev/null @@ -1,47 +0,0 @@ -const fs = require("fs-extra"); -const path = require("path"); -const { safeSymlink } = require("../common/symlink"); - -const [ - nodePath, - scriptPath, - targetLabel, - joinedInternalDeps, - joinedSrcs, - destinationDir -] = process.argv; - -const internalDeps = joinedInternalDeps.split("|"); -const srcs = joinedSrcs.split("|"); - -fs.mkdirSync(destinationDir); - -// Copy every internal dependency into the appropriate location. -for (const internalDep of internalDeps) { - if (!internalDep) { - continue; - } - const [joinedSrcs, compiledDir] = internalDep.split(":"); - const srcs = joinedSrcs.split(";"); - for (const src of srcs) { - if (!src) { - continue; - } - safeSymlink(path.join(compiledDir, src), path.join(destinationDir, src)); - } -} - -// Copy source code. -for (const src of srcs) { - if (!src) { - continue; - } - if (!fs.existsSync(src)) { - console.error(` -Missing file ${src} required by ${targetLabel}. -`); - process.exit(1); - } - const destinationFilePath = path.join(destinationDir, src); - safeSymlink(src, destinationFilePath); -} From 71e4ce38ec291f86bb5f1feabf64f0c6dd00c89e Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:23:36 -0800 Subject: [PATCH 37/43] Moving js_binary to a BazelAction --- internal/js_binary/compile.js | 104 +++++++++++++++++----------------- internal/js_binary/rule.bzl | 22 ++++++- 2 files changed, 74 insertions(+), 52 deletions(-) diff --git a/internal/js_binary/compile.js b/internal/js_binary/compile.js index d606c08..8ea7771 100644 --- a/internal/js_binary/compile.js +++ b/internal/js_binary/compile.js @@ -1,58 +1,60 @@ -const child_process = require("child_process"); -const fs = require("fs-extra"); +const { BazelAction } = require("../common/actions/run_js/BazelAction"); const path = require("path"); const webpack = require("webpack"); -const [ - nodePath, - scriptPath, - libBuildfilePath, - entry, - mode, - installedNpmPackagesDir, - compiledDir, - outputFile -] = process.argv; - -webpack( - { - entry: path.resolve( - path.join(compiledDir, path.dirname(libBuildfilePath), entry) - ), - output: { - filename: path.basename(outputFile), - path: path.resolve(path.dirname(outputFile)) - }, +BazelAction( + {}, + async ({ + srcDir, + outDir, + libBuildfilePath, + entry, mode, - target: "node", - resolve: { - modules: [ - path.resolve(path.join(installedNpmPackagesDir, "node_modules")) - ] - }, - plugins: [ - new webpack.BannerPlugin({ - banner: "#!/usr/bin/env node", - raw: true - }) - ] - }, - (err, stats) => { - // See https://webpack.js.org/api/node/#error-handling. - if (err) { - console.error(err.stack || err); - if (err.details) { - console.error(err.details); + installedNpmPackagesDir, + compiledDir, + outputFile + }) => { + webpack( + { + entry: path.resolve( + path.join(compiledDir, path.dirname(libBuildfilePath), entry) + ), + output: { + filename: path.basename(outputFile), + path: path.resolve(path.dirname(outputFile)) + }, + mode, + target: "node", + resolve: { + modules: [ + path.resolve(path.join(installedNpmPackagesDir, "node_modules")) + ] + }, + plugins: [ + new webpack.BannerPlugin({ + banner: "#!/usr/bin/env node", + raw: true + }) + ] + }, + (err, stats) => { + // See https://webpack.js.org/api/node/#error-handling. + if (err) { + console.error(err.stack || err); + if (err.details) { + console.error(err.details); + } + process.exit(1); + } + const info = stats.toJson(); + if (stats.hasErrors()) { + console.error(info.errors); + process.exit(1); + } + if (stats.hasWarnings()) { + console.warn(info.warnings); + } } - process.exit(1); - } - const info = stats.toJson(); - if (stats.hasErrors()) { - console.error(info.errors); - process.exit(1); - } - if (stats.hasWarnings()) { - console.warn(info.warnings); - } + ); } ); diff --git a/internal/js_binary/rule.bzl b/internal/js_binary/rule.bzl index d39383c..596efd7 100644 --- a/internal/js_binary/rule.bzl +++ b/internal/js_binary/rule.bzl @@ -1,7 +1,27 @@ -load("//internal/js_library:rule.bzl", "JsLibraryInfo") load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") +load("//internal/common:context.bzl", "JS_LIBRARY_ATTRIBUTES", "js_context") def _js_binary_impl(ctx): + js = js_context(ctx) + providers = [] + + compile_args = js.script_args(js) + # Run `node js_binary/compile.js`. + ctx.file._js_binary_compile_script.path, + # Path of the directory containing the lib's BUILD.bazel file. + ctx.attr.lib[JsLibraryInfo].build_file_path, + # Entry point for Webpack (e.g. "main.ts"). + ctx.attr.entry, + # Mode for Webpack. + ctx.attr.mode, + # Directory containing external NPM dependencies the code depends on. + ctx.attr.lib[JsLibraryInfo].npm_packages_installed_dir.path, + # Directory containing the compiled source code of the js_library. + ctx.attr.lib[JsLibraryInfo].compiled_javascript_dir.path, + # Directory in which to place the compiled JavaScript. + ctx.outputs.executable_file.path, + + ctx.actions.run( inputs = [ ctx.file._js_binary_compile_script, From b4113017c43ee3cff4d9fa4e73a874f236df60d4 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:44:55 -0800 Subject: [PATCH 38/43] Fix missing colon --- internal/common/context.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 4263ff4..fca82a7 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -84,7 +84,7 @@ JS_CONTEXT_ATTRIBUTES = { "_common_js_files": attr.label( allow_files = True, default = Label("//internal/common:_common_js_files") - ) + ), "_actions_bazel_action_js": attr.label( allow_files = True, single_file = True, From c8aa5769fffe4447f39ff3372c42dcf007c2db38 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:45:40 -0800 Subject: [PATCH 39/43] Fix references to JsLibraryInfo and JsModuleInfo --- defs.bzl | 3 ++- internal/js_binary/rule.bzl | 17 +---------------- internal/js_module/rule.bzl | 8 ++------ internal/js_script_and_test/rule.bzl | 2 +- internal/ts_library/rule.bzl | 1 - internal/web_bundle/rule.bzl | 3 +-- 6 files changed, 7 insertions(+), 27 deletions(-) diff --git a/defs.bzl b/defs.bzl index 5d0416b..52a94b2 100644 --- a/defs.bzl +++ b/defs.bzl @@ -1,4 +1,5 @@ -load("//internal/js_library:rule.bzl", "JsLibraryInfo", "js_library") +load("//internal/common:context.bzl", "JsLibraryInfo", "JsModuleInfo") +load("//internal/js_library:rule.bzl", "js_library") load("//internal/ts_library:rule.bzl", "ts_library") load("//internal/js_module:rule.bzl", "js_module") load("//internal/js_binary:rule.bzl", "js_binary") diff --git a/internal/js_binary/rule.bzl b/internal/js_binary/rule.bzl index 596efd7..820e80a 100644 --- a/internal/js_binary/rule.bzl +++ b/internal/js_binary/rule.bzl @@ -1,26 +1,11 @@ load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") -load("//internal/common:context.bzl", "JS_LIBRARY_ATTRIBUTES", "js_context") +load("//internal/common:context.bzl", "JsLibraryInfo", "JS_LIBRARY_ATTRIBUTES", "js_context") def _js_binary_impl(ctx): js = js_context(ctx) providers = [] compile_args = js.script_args(js) - # Run `node js_binary/compile.js`. - ctx.file._js_binary_compile_script.path, - # Path of the directory containing the lib's BUILD.bazel file. - ctx.attr.lib[JsLibraryInfo].build_file_path, - # Entry point for Webpack (e.g. "main.ts"). - ctx.attr.entry, - # Mode for Webpack. - ctx.attr.mode, - # Directory containing external NPM dependencies the code depends on. - ctx.attr.lib[JsLibraryInfo].npm_packages_installed_dir.path, - # Directory containing the compiled source code of the js_library. - ctx.attr.lib[JsLibraryInfo].compiled_javascript_dir.path, - # Directory in which to place the compiled JavaScript. - ctx.outputs.executable_file.path, - ctx.actions.run( inputs = [ diff --git a/internal/js_module/rule.bzl b/internal/js_module/rule.bzl index 66d5a4b..35deb74 100644 --- a/internal/js_module/rule.bzl +++ b/internal/js_module/rule.bzl @@ -1,11 +1,7 @@ -load("//internal/js_library:rule.bzl", "JsLibraryInfo") - -JsModuleInfo = provider(fields = [ - "name", - "single_file", -]) +load("//internal/common:context.bzl", "JsLibraryInfo", "JsModuleInfo", "JS_LIBRARY_ATTRIBUTES", "js_context") def _js_module_impl(ctx): + return [ ctx.attr.lib[JsLibraryInfo], JsModuleInfo( diff --git a/internal/js_script_and_test/rule.bzl b/internal/js_script_and_test/rule.bzl index 16b2347..26be53c 100644 --- a/internal/js_script_and_test/rule.bzl +++ b/internal/js_script_and_test/rule.bzl @@ -1,5 +1,5 @@ -load("//internal/js_library:rule.bzl", "JsLibraryInfo") load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") +load("//internal/common:context.bzl", "JsLibraryInfo", "JsModuleInfo", "JS_LIBRARY_ATTRIBUTES", "js_context") def _js_script_impl(ctx): # Create a directory that contains: diff --git a/internal/ts_library/rule.bzl b/internal/ts_library/rule.bzl index d25b021..8d6f028 100644 --- a/internal/ts_library/rule.bzl +++ b/internal/ts_library/rule.bzl @@ -1,4 +1,3 @@ -load("//internal/js_library:rule.bzl", "JsLibraryInfo") load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") load("//internal/common:context.bzl", "JS_LIBRARY_ATTRIBUTES", "js_context") diff --git a/internal/web_bundle/rule.bzl b/internal/web_bundle/rule.bzl index 7ec7475..926d3af 100644 --- a/internal/web_bundle/rule.bzl +++ b/internal/web_bundle/rule.bzl @@ -1,6 +1,5 @@ -load("//internal/js_library:rule.bzl", "JsLibraryInfo") -load("//internal/js_module:rule.bzl", "JsModuleInfo") load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") +load("//internal/common:context.bzl", "JsLibraryInfo", "JsModuleInfo", "JS_LIBRARY_ATTRIBUTES", "js_context") def _web_bundle_impl(ctx): webpack_config = _create_webpack_config(ctx) From 21e2ebfdb795e49172219c2f541b781885e5caf6 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:46:20 -0800 Subject: [PATCH 40/43] Move js_library to new shared context --- internal/js_library/compile.js | 96 ++++++++-------- internal/js_library/rule.bzl | 204 ++++----------------------------- 2 files changed, 72 insertions(+), 228 deletions(-) diff --git a/internal/js_library/compile.js b/internal/js_library/compile.js index 37a7e31..caef5a4 100644 --- a/internal/js_library/compile.js +++ b/internal/js_library/compile.js @@ -1,58 +1,56 @@ +const { + BazelAction, + safeSymlink +} = require("../common/actions/run_js/BazelAction"); +const child_process = require("child_process"); const fs = require("fs-extra"); const path = require("path"); const babel = require("babel-core"); -const { safeSymlink } = require("../common/symlink"); -const [ - nodePath, - scriptPath, - fullSrcDir, - destinationDir, - joinedSrcs -] = process.argv; +BazelAction({}, async ({ srcDir, outDir }) => { + const srcs = new Set(joinedSrcs.split("|")); -const srcs = new Set(joinedSrcs.split("|")); - -// Compile with Babel. -transformDir("."); - -function transformDir(dirRelativePath) { - for (const fileName of fs.readdirSync( - path.join(fullSrcDir, dirRelativePath) - )) { - const relativeFilePath = path.join(dirRelativePath, fileName); - const srcFilePath = path.join(fullSrcDir, relativeFilePath); - let destFilePath = path.join(destinationDir, relativeFilePath); - fs.ensureDirSync(path.dirname(destFilePath)); - if (fs.lstatSync(srcFilePath).isDirectory()) { - transformDir(relativeFilePath); - } else if ( - srcs.has(relativeFilePath) && - (fileName.endsWith(".es6") || + function transformDir(dirRelativePath) { + currentDir = path.join(srcDir, dirRelativePath); + for (const fileName of fs.readdirSync(currentDir)) { + const relativeFilePath = path.join(dirRelativePath, fileName); + const srcFilePath = path.join(srcDir, relativeFilePath); + let destFilePath = path.join(outDir, relativeFilePath); + fs.ensureDirSync(path.dirname(destFilePath)); + if (fs.lstatSync(srcFilePath).isDirectory()) { + transformDir(relativeFilePath); + } else if ( + fileName.endsWith(".es6") || fileName.endsWith(".js") || - fileName.endsWith(".jsx")) - ) { - const transformed = babel.transformFileSync(srcFilePath, { - plugins: [ - "transform-decorators-legacy", - "transform-es2015-modules-commonjs" - ], - presets: ["env", "stage-2", "react"], - ignore: "node_modules" - }); - if (!transformed.code) { - throw new Error(`Could not compile ${srcFilePath}.`); - } - if (!destFilePath.endsWith(".js")) { - destFilePath = - destFilePath.substr(0, destFilePath.lastIndexOf(".")) + ".js"; + fileName.endsWith(".jsx") + ) { + const transformed = babel.transformFileSync(srcFilePath, { + plugins: [ + "transform-decorators-legacy", + "transform-es2015-modules-commonjs" + ], + presets: ["env", "stage-2", "react"], + ignore: "node_modules" + }); + if (!transformed.code) { + throw new Error(`Could not compile ${srcFilePath}.`); + } + if (!destFilePath.endsWith(".js")) { + destFilePath = + destFilePath.substr(0, destFilePath.lastIndexOf(".")) + ".js"; + } + fs.writeFileSync(destFilePath, transformed.code, "utf8"); + } else { + // Symlink any file that: + // - isn't a source file of this package; or + // - is not a JavaScript file (e.g. CSS assets). + safeSymlink(srcFilePath, destFilePath); } - fs.writeFileSync(destFilePath, transformed.code, "utf8"); - } else { - // Symlink any file that: - // - isn't a source file of this package; or - // - is not a JavaScript file (e.g. CSS assets). - safeSymlink(srcFilePath, destFilePath); } } -} + + // Compile with Babel. + transformDir("."); +}); + +const [nodePath, scriptPath, fullSrcDir, outDir, joinedSrcs] = process.argv; diff --git a/internal/js_library/rule.bzl b/internal/js_library/rule.bzl index e1af732..a64c1c4 100644 --- a/internal/js_library/rule.bzl +++ b/internal/js_library/rule.bzl @@ -1,189 +1,38 @@ -load("//internal/npm_packages:rule.bzl", "NpmPackagesInfo") - -JsLibraryInfo = provider(fields = [ - # Path of the BUILD.bazel file relative to the workspace root. - "build_file_path", - # Directory containing the JavaScript files (and potentially other assets). - "compiled_javascript_dir", - # Source files provided as input. - "javascript_source_files", - # Other js_library targets depended upon. - "internal_deps", - # Depset of npm_packages depended upon (at most one element). - "npm_packages", - # Directory in which node_modules/ with external NPM packages can be found. - "npm_packages_installed_dir", -]) +load("//internal/common:context.bzl", "JS_LIBRARY_ATTRIBUTES", "js_context") def _js_library_impl(ctx): - # Ensure that we depend on at most one npm_packages, since we don't want to - # have conflicting package versions coming from separate node_modules - # directories. - direct_npm_packages = [ - dep - for dep in ctx.attr.deps - if NpmPackagesInfo in dep - ] - if len(direct_npm_packages) > 1: - fail("Found more than one set of NPM packages in target definition: " + ",".join([ - dep.label - for dep in direct_npm_packages - ])) - extended_npm_packages = depset( - direct = direct_npm_packages, - transitive = [ - dep[JsLibraryInfo].npm_packages - for dep in ctx.attr.deps - if JsLibraryInfo in dep - ], - ) - npm_packages_list = extended_npm_packages.to_list() - if len(npm_packages_list) > 1: - fail("Found more than one set of NPM packages through dependencies: " + ",".join([ - dep.label - for dep in npm_packages_list - ])) + js = js_context(ctx) + providers = [] - # If we depend on an npm_packages target, we'll use its node_modules - # directory to find modules. Otherwise, we'll use an empty node_modules - # directory. - npm_packages = ( - npm_packages_list[0] if len(npm_packages_list) == 1 else ctx.attr._empty_npm_packages - ) + js_library = js.library_info(js) + providers.append(js_library) - # Gather all internal deps (other js_library rules). - internal_deps = depset( - direct = [ - dep - for dep in ctx.attr.deps - if JsLibraryInfo in dep - ], - transitive = [ - dep[JsLibraryInfo].internal_deps - for dep in ctx.attr.deps - if JsLibraryInfo in dep - ], - ) + js_source = js.library_to_source_info(js, js_library) + + js.create_source_dir(js, js_source, ctx.outputs.compilation_src_dir) - # Create a directory that contains: - # - source files (including all internal dependencies) - # - node_modules (symlinked to installed external dependencies directory) - _js_library_create_full_src( - ctx, - internal_deps, - npm_packages, - ) - _js_library_compile( - ctx, - internal_deps, - npm_packages, - ) - return [ - JsLibraryInfo( - build_file_path = ctx.build_file_path, - javascript_source_files = [_compiled_extension(f.path) for f in ctx.files.srcs], - compiled_javascript_dir = ctx.outputs.compiled_dir, - internal_deps = internal_deps, - npm_packages = extended_npm_packages, - npm_packages_installed_dir = npm_packages[NpmPackagesInfo].installed_dir, - ), - ] + compile_args = js.script_args(js) + compile_args.add("--srcDir", ctx.outputs.compilation_src_dir) + compile_args.add("--outDir", ctx.outputs.compiled_dir) + compile_args.add_all(js_library.all_sources) -def _compiled_extension(path): - if path.endswith(".es6") or path.endswith(".jsx"): - return path[:-4] + ".js" - else: - return path - -def _js_library_create_full_src(ctx, internal_deps, npm_packages): - ctx.actions.run( - inputs = [ - ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, - ctx.file._js_library_create_full_src_script, - npm_packages[NpmPackagesInfo].installed_dir, - ] + [ - d[JsLibraryInfo].compiled_javascript_dir - for d in internal_deps - ] + ctx.files.srcs, - outputs = [ctx.outputs.compiled_javascript_dir], - executable = ctx.file._internal_nodejs, - env = { - "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", - }, - arguments = [ - # Run `node process.js`. - ctx.file._js_library_create_full_src_script.path, - # Label of the build target (for helpful errors). - "//" + npm_packages.label.package + ":" + npm_packages.label.name, - # Source directories of the js_library targets we depend on. - ("|".join([ - (";".join(d[JsLibraryInfo].javascript_source_files)) + ":" + - d[JsLibraryInfo].compiled_javascript_dir.path - for d in internal_deps - ])), - # List of source files, which will be processed ("import" statements - # automatically replaced) and copied into the new directory. - ("|".join([ - f.path - for f in ctx.files.srcs - ])), - # Directory in which to place the result. - ctx.outputs.compiled_javascript_dir.path, - ], + js_compile_inputs = depset( + direct = [ctx.outputs.compilation_src_dir], + transitive = [js_library.all_sources], ) -def _js_library_compile(ctx, internal_deps, npm_packages): - ctx.actions.run( - inputs = [ - ctx.file._js_library_compile_script, - ctx.outputs.compiled_javascript_dir, - ctx.attr._internal_packages[NpmPackagesInfo].installed_dir, - npm_packages[NpmPackagesInfo].installed_dir, - ] + [ - d[JsLibraryInfo].compiled_javascript_dir - for d in internal_deps - ], + js.run_js( + js, + inputs = js_compile_inputs, outputs = [ctx.outputs.compiled_dir], - executable = ctx.file._internal_nodejs, - env = { - "NODE_PATH": ctx.attr._internal_packages[NpmPackagesInfo].installed_dir.path + "/node_modules", - }, - arguments = [ - # Run `node js_library/compile.js`. - ctx.file._js_library_compile_script.path, - # Directory in which the source code can be found. - ctx.outputs.compiled_javascript_dir.path, - # Directory in which to output the compiled JavaScript. - ctx.outputs.compiled_dir.path, - # List of source files, excluding source files from dependencies. - ("|".join([ - f.path - for f in ctx.files.srcs - ])), - ], + script = ctx.file._js_library_compile_script, + script_args = compile_args, ) + return providers + js_library = rule( - attrs = { - "srcs": attr.label_list( - allow_files = True, - mandatory = True, - ), - "deps": attr.label_list( - providers = [ - [JsLibraryInfo], - [NpmPackagesInfo], - ], - default = [], - ), - "_internal_packages": attr.label( - default = Label("//internal:packages"), - ), - "_internal_nodejs": attr.label( - allow_files = True, - single_file = True, - default = Label("@nodejs//:node"), - ), + attrs = dict(JS_LIBRARY_ATTRIBUTES, **{ "_js_library_create_full_src_script": attr.label( allow_files = True, single_file = True, @@ -194,13 +43,10 @@ js_library = rule( single_file = True, default = Label("//internal/js_library:compile.js"), ), - "_empty_npm_packages": attr.label( - default = Label("//internal/npm_packages/empty:packages"), - ), - }, + }), outputs = { + "compilation_src_dir": "%{name}_compilation_src", "compiled_dir": "%{name}_compiled", - "compiled_javascript_dir": "%{name}_full_src", }, implementation = _js_library_impl, ) From 7d336aa20713a4f06efd873176c8c0ba90ff06f2 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:49:13 -0800 Subject: [PATCH 41/43] Fixing incorrect tag on NODE_MODULE_MARKER --- internal/common/context.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index fca82a7..92f8633 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -172,7 +172,7 @@ def _js_library_info(js, attr = None): dep_js_module = dep[JsModuleInfo] transitive_dep_modules.append(dep_js_module.all_dep_modules) transitive_dep_module_targets.append(dep_js_module.all_dep_module_targets) - elif hasattr(dep, "tags") and "NODE_MODULES_MARKER" in getattr(dep, "tags"): + elif hasattr(dep, "tags") and "NODE_MODULE_MARKER" in getattr(dep, "tags"): # The dependency is a module defined by rules_nodejs direct_modules += dep From f165b7d1ab7da9cff9bc5a64b87fca147eda4e90 Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 15:55:09 -0800 Subject: [PATCH 42/43] Just a little better documentation on generator scripts --- internal/common/context.bzl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index 92f8633..b22f516 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -43,7 +43,8 @@ JsSourceInfo = provider(fields = [ "js", # The source js files "src_files", - # List of scripts to output + # List of scripts that will be used to generate files in the source directory + # For an example of these scripts see internal/ts_library/tsconfig.gen.js "gen_scripts", # The library rule that generated this source "library", From 655b9b29b269a1ac47175138ade3d64803aa54ce Mon Sep 17 00:00:00 2001 From: Rich Adams Date: Sun, 9 Dec 2018 20:22:35 -0800 Subject: [PATCH 43/43] Updating documentation strings --- internal/common/context.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/common/context.bzl b/internal/common/context.bzl index b22f516..ba59f59 100644 --- a/internal/common/context.bzl +++ b/internal/common/context.bzl @@ -217,6 +217,10 @@ def _library_to_source_info(js, library, gen_scripts = None): """Create a JsSource provider for a given library The library is a target, but this resolves the actual source files needed to build the js library. + + The gen_scripts is a list of js files that export a single function whose output + generates a list of files: + modules.exports = async ({ package, into, inputs}) """ return JsSourceInfo( @@ -241,7 +245,6 @@ def _script_args(js): Args: js: JsContext object - script_file: File object for the script to be run """ args = js.actions.args()