diff --git a/src/flast.js b/src/flast.js index 2cf91a0..2186d5f 100644 --- a/src/flast.js +++ b/src/flast.js @@ -38,15 +38,6 @@ const generateFlatASTDefaultOptions = { }, }; -/** - * Return a function which retrieves a node's source on demand - * @param {string} src - * @returns {function(number, number): string} - */ -function createSrcClosure(src) { - return function(start, end) {return src.slice(start, end);}; -} - /** * @param {string} inputCode * @param {object} opts Optional changes to behavior. See generateFlatASTDefaultOptions for available options. @@ -96,13 +87,13 @@ function generateRootNode(inputCode, opts = {}) { let rootNode; try { rootNode = parseCode(inputCode, parseOpts); - if (opts.includeSrc) rootNode.srcClosure = createSrcClosure(inputCode); + if (opts.includeSrc) rootNode.src = inputCode; } catch (e) { // If any parse error occurs and alternateSourceTypeOnFailure is set, try 'script' mode if (opts.alternateSourceTypeOnFailure) { try { rootNode = parseCode(inputCode, {...parseOpts, sourceType: 'script'}); - if (opts.includeSrc) rootNode.srcClosure = createSrcClosure(inputCode); + if (opts.includeSrc) rootNode.src = inputCode; } catch (e2) { logger.debug('Failed to parse as module and script:', e, e2); } @@ -113,6 +104,63 @@ function generateRootNode(inputCode, opts = {}) { return rootNode; } +/** + * @param {object} opts + * @param {ASTNode} rootNode + * @param {{number: ASTScope}} scopes + * @param {number} nodeId + * @param {ASTNode} node + * @return {ASTNode} + */ +function parseNode (opts, rootNode, scopes, nodeId, node) { + if (node.nodeId) return; + node.childNodes = node.childNodes || []; + const childrenLoc = {}; // Store the location of child nodes to sort them by order + node.parentKey = node.parentKey || ''; // Make sure parentKey exists + // Iterate over all keys of the node to find child nodes + const keys = Object.keys(node); + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + if (excludedParentKeys.includes(key)) continue; + const content = node[key]; + if (content && typeof content === 'object') { + // Sort each child node by its start position + // and set the parentNode and parentKey attributes + if (Array.isArray(content)) { + for (let j = 0; j < content.length; j++) { + const childNode = content[j]; + if (!childNode) continue; + childNode.parentNode = node; + childNode.parentKey = key; + childrenLoc[childNode.start] = childNode; + } + } else { + content.parentNode = node; + content.parentKey = key; + childrenLoc[content.start] = content; + } + } + } + // Add the child nodes to top of the stack and populate the node's childNodes array + node.childNodes.push(...Object.values(childrenLoc)); + + node.nodeId = nodeId; + if (opts.detailed) { + node.scope = scopes[node.scopeId] || node.parentNode?.scope; + node.lineage = [...node.parentNode?.lineage || []]; + if (!node.lineage.includes(node.scope.scopeId)) { + node.lineage.push(node.scope.scopeId); + } + } + // Avoid using a getter with a closure around source here, as the + // memory requirement for a function per node is far greater than using + // a string reference for sufficiently large AST Trees + // (~2.4 nodes for 3 Gib). + if (opts.includeSrc && !node.src) + node.src = rootNode.src.substring(node.start,node.end); + return node; +} + /** * @param rootNode * @param opts @@ -120,62 +168,29 @@ function generateRootNode(inputCode, opts = {}) { */ function extractNodesFromRoot(rootNode, opts) { opts = {...generateFlatASTDefaultOptions, ...opts}; - let nodeId = 0; const typeMap = {}; const allNodes = []; const scopes = opts.detailed ? getAllScopes(rootNode) : {}; - const stack = [rootNode]; - while (stack.length) { - const node = stack.shift(); - if (node.nodeId) continue; - node.childNodes = node.childNodes || []; - const childrenLoc = {}; // Store the location of child nodes to sort them by order - node.parentKey = node.parentKey || ''; // Make sure parentKey exists - // Iterate over all keys of the node to find child nodes - const keys = Object.keys(node); - for (let i = 0; i < keys.length; i++) { - const key = keys[i]; - if (excludedParentKeys.includes(key)) continue; - const content = node[key]; - if (content && typeof content === 'object') { - // Sort each child node by its start position - // and set the parentNode and parentKey attributes - if (Array.isArray(content)) { - for (let j = 0; j < content.length; j++) { - const childNode = content[j]; - if (!childNode) continue; - childNode.parentNode = node; - childNode.parentKey = key; - childrenLoc[childNode.start] = childNode; - } - } else { - content.parentNode = node; - content.parentKey = key; - childrenLoc[content.start] = content; - } - } + let nodeId = 0; + let visitor = rootNode; + const lastParsed = {}; + while (visitor) { + if(!visitor.childNodes){ + allNodes.push(parseNode(opts, rootNode, scopes, nodeId++, visitor)); + typeMap[visitor.type] = typeMap[visitor.type] || []; + typeMap[visitor.type].push(visitor); + lastParsed[visitor.nodeId] = 0; } - // Add the child nodes to top of the stack and populate the node's childNodes array - stack.unshift(...Object.values(childrenLoc)); - node.childNodes.push(...Object.values(childrenLoc)); - - allNodes.push(node); - node.nodeId = nodeId++; - typeMap[node.type] = typeMap[node.type] || []; - typeMap[node.type].push(node); - if (opts.detailed) { - node.scope = scopes[node.scopeId] || node.parentNode?.scope; - node.lineage = [...node.parentNode?.lineage || []]; - if (!node.lineage.includes(node.scope.scopeId)) { - node.lineage.push(node.scope.scopeId); - } + let visitorId = visitor.nodeId; + if(lastParsed[visitorId] < visitor.childNodes.length){ + visitor = visitor.childNodes[lastParsed[visitorId]++]; + }else{ + visitor = visitor.parentNode; + delete lastParsed[visitorId]; } - // Add a getter for the node's source code - if (opts.includeSrc && !node.src) Object.defineProperty(node, 'src', { - get() {return rootNode.srcClosure(node.start, node.end);}, - }); } + if (opts.detailed) { const identifiers = typeMap.Identifier || []; const scopeVarMaps = buildScopeVarMaps(scopes);