Update changelog for version 2.2.0 and improve package cache handling

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### Version 2.2.0
+
+- Performance improvement to cache package information
+
 ### Version 2.1.6
 
 - Remove "Reduce Config Files" browserslist tip

package.json

@@ -2,7 +2,7 @@
   "name": "webnative",
   "displayName": "WebNative",
   "description": "Create and maintain web and native projects",
-  "version": "2.1.6",
+  "version": "2.2.0",
   "whatsNewRevision": 1,
   "icon": "media/webnative.png",
   "publisher": "WebNative",

src/context-variables.ts

@@ -93,3 +93,10 @@ export function PackageCacheModified(project: Project) {
 }
 
 export const LastManifestCheck = 'LastManifestCheck';
+
+export function PackageCacheRefreshedAt(project: Project) {
+  if (project?.monoRepo?.localPackageJson) {
+    return 'packageRefreshedAt_' + project.monoRepo.name;
+  }
+  return 'packageRefreshedAt';
+}

src/process-packages.ts

@@ -1,7 +1,7 @@
 import { coerce } from 'semver';
 import { Command, Tip, TipType } from './tip';
 import { Project } from './project';
-import { getRunOutput, isWindows, stripJSON, tEnd, tStart } from './utilities';
+import { getRunOutput, isWindows, stripJSON } from './utilities';
 import { NpmDependency, NpmOutdatedDependency, NpmPackage, PackageType, PackageVersion } from './npm-model';
 import { listCommand, outdatedCommand } from './node-commands';
 import {
@@ -10,6 +10,7 @@ import {
   PackageCacheList,
   PackageCacheModified,
   PackageCacheOutdated,
+  PackageCacheRefreshedAt,
 } from './context-variables';
 import { join } from 'path';
 import { exState } from './tree-provider';
@@ -68,6 +69,53 @@ async function runListPackages(project: Project, folder: string, context: Extens
   }
 }
 
+/**
+ * Fetches fresh outdated and list data from npm/yarn and updates the workspace cache.
+ * Safe to call without await for background refresh.
+ */
+async function refreshPackageData(project: Project, folder: string, context: ExtensionContext): Promise<void> {
+  const outdatedCmd = outdatedCommand(project);
+  try {
+    const [, freshVersions] = await Promise.all([
+      getRunOutput(outdatedCmd, folder, undefined, true, true)
+        .then((data) => {
+          if (project.isYarnV1()) {
+            data = fixYarnV1Outdated(data, project.packageManager);
+          } else if (project.isModernYarn()) {
+            data = fixYarnOutdated(data, project);
+          }
+          context.workspaceState.update(PackageCacheOutdated(project), data);
+        })
+        .catch((reason) => {
+          write(`> ${outdatedCmd}`);
+          writeError(reason);
+        }),
+      runListPackages(project, folder, context),
+    ]);
+    if (freshVersions) {
+      context.workspaceState.update(PackageCacheList(project), freshVersions);
+    } else {
+      context.workspaceState.update(PackageCacheList(project), '{}');
+    }
+    context.workspaceState.update(PackageCacheModified(project), project.modified.toUTCString());
+    context.workspaceState.update(PackageCacheRefreshedAt(project), Date.now());
+  } catch (err) {
+    if (err && err.includes('401')) {
+      window.showInformationMessage(
+        `Unable to run '${outdatedCommand(project)}' due to authentication error. Check .npmrc`,
+        'OK',
+      );
+    } else if (project.isModernYarn()) {
+      writeWarning(
+        `Modern Yarn does not have a command to review outdated package versions. Most functionality of this extension will be disabled.`,
+      );
+    } else {
+      writeError(`Unable to run '${outdatedCommand(project)}'. Try reinstalling node modules.`);
+      console.error(err);
+    }
+  }
+}
+
 export async function processPackages(
   folder: string,
   allDependencies: object,
@@ -78,7 +126,6 @@ export async function processPackages(
   if (!lstatSync(folder).isDirectory()) {
     return {};
   }
-
   // npm outdated only shows dependencies and not dev dependencies if the node module isn't installed
   let outdated = '[]';
   let versions = '{}';
@@ -91,43 +138,34 @@ export async function processPackages(
     if (changed) {
       exState.syncDone = [];
     }
-    if (changed || !outdated || !versions) {
-      const outdatedCmd = outdatedCommand(project);
-
-      const values = await Promise.all([
-        getRunOutput(outdatedCmd, folder, undefined, true, true)
-          .then((data) => {
-            if (project.isYarnV1()) {
-              data = fixYarnV1Outdated(data, project.packageManager);
-            } else if (project.isModernYarn()) {
-              data = fixYarnOutdated(data, project);
-            }
-            outdated = data;
-            context.workspaceState.update(PackageCacheOutdated(project), outdated);
-          })
-          .catch((reason) => {
-            write(`> ${outdatedCmd}`);
-            writeError(reason);
-          }),
-        runListPackages(project, folder, context),
-      ]);
-      versions = values[1];
-      context.workspaceState.update(PackageCacheList(project), versions);
-      context.workspaceState.update(PackageCacheModified(project), packagesModified.toUTCString());
-    } else {
-      // Use the cached value
-      // But also get a copy of the latest packages for updating later
-      const itsAGoodTime = false;
-      if (itsAGoodTime) {
-        getRunOutput(outdatedCommand(project), folder, undefined, true).then((outdatedFresh) => {
-          context.workspaceState.update(PackageCacheOutdated(project), outdatedFresh);
-          context.workspaceState.update(PackageCacheModified(project), packagesModified.toUTCString());
-        });
 
-        getRunOutput(listCommand(project), folder, undefined, true).then((versionsFresh) => {
-          context.workspaceState.update(PackageCacheList(project), versionsFresh);
-        });
-      }
+    const hasCachedData = outdated !== undefined && versions !== undefined;
+    const lastRefreshedAt: number = context.workspaceState.get(PackageCacheRefreshedAt(project)) ?? 0;
+    const oneHourMs = 60 * 60 * 1000;
+    const refreshIsStale = Date.now() - lastRefreshedAt > oneHourMs;
+    // versions with length <= 2 is '{}' or '' - treat as unusable, needs a refresh
+    const versionsUnusable = !versions || versions.length <= 2;
+
+    write(
+      `[processPackages] key=${PackageCacheOutdated(project)} outdated=${outdated === undefined ? 'undefined' : outdated === null ? 'null' : 'set(len=' + String(outdated).length + ')'} versions=${versions === undefined ? 'undefined' : versions === null ? 'null' : 'set(len=' + String(versions).length + ')'} changed=${changed} stale=${refreshIsStale} hasCachedData=${hasCachedData}`,
+    );
+
+    if (!hasCachedData) {
+      // No cache at all - must block and wait for fresh data on first run
+      await refreshPackageData(project, folder, context);
+      outdated = context.workspaceState.get(PackageCacheOutdated(project)) ?? '[]';
+      versions = context.workspaceState.get(PackageCacheList(project)) ?? '{}';
+    } else if (changed || refreshIsStale || versionsUnusable) {
+      // Cache exists but package.json changed, data is over 1 hour old, or versions is empty -
+      // return stale cache now and refresh in background
+      write(
+        `[processPackages] Returning cached data, refreshing in background (changed=${changed}, stale=${refreshIsStale}, versionsUnusable=${versionsUnusable})`,
+      );
+      refreshPackageData(project, folder, context).catch((err) =>
+        console.error('Background package refresh failed', err),
+      );
+    } else {
+      write(`[processPackages] Using cached data as-is`);
     }
   } catch (err) {
     outdated = '[]';
@@ -151,17 +189,13 @@ export async function processPackages(
   // outdated is an array with:
   //  "@ionic-native/location-accuracy": { "wanted": "5.36.0", "latest": "5.36.0", "dependent": "cordova-old" }
 
-  tStart('processDependencies');
   const packages = processDependencies(
     allDependencies,
     getOutdatedData(outdated),
     devDependencies,
     getListData(versions),
   );
-  tEnd('processDependencies');
-  tStart('inspectPackages');
   inspectPackages(project.projectFolder() ? project.projectFolder() : folder, packages);
-  tEnd('inspectPackages');
   return packages;
 }
 

src/project.ts

@@ -695,10 +695,8 @@ export async function inspectProject(
   }
 
   guessFramework(project);
-
   checkNodeVersion();
   project.getIgnored(context);
-
   await getRecommendations(project, context, packages);
 
   commands.executeCommand(VSCommand.setContext, Context.inspectedProject, true);

src/recommend.ts

@@ -22,7 +22,7 @@ import { exState } from './tree-provider';
 import { getAndroidWebViewList } from './android-debug-list';
 import { getDebugBrowserName } from './preview';
 import { checkIonicNativePackages } from './rules-ionic-native';
-import { alt, getRunOutput, showProgress, tEnd, tStart } from './utilities';
+import { alt, getRunOutput, showProgress } from './utilities';
 import { startStopLogServer } from './log-server';
 import { getBuildConfigurationName } from './build-configuration';
 import { liveReloadSSL } from './live-reload';
@@ -53,8 +53,6 @@ function hasWebPackages() {
 }
 
 export async function getRecommendations(project: Project, context: ExtensionContext, packages: any): Promise<void> {
-  tStart('getRecommendations');
-
   const isWebProjectOnly = !exists('@capacitor/core') && hasWebPackages();
 
   if (project.isCapacitor || isWebProjectOnly) {
@@ -341,20 +339,13 @@ export async function getRecommendations(project: Project, context: ExtensionCon
     }
   }
 
-  tEnd('getRecommendations');
-
   if (project.isCapacitor) {
-    tStart('checkCapacitorRules');
     await checkCapacitorRules(project, context);
-    tEnd('checkCapacitorRules');
-    tStart('capacitorRecommendations');
     checkIonicNativePackages(packages, project);
     checkCordovaPlugins(packages, project);
     project.tips(await capacitorRecommendations(project, false));
-    tEnd('capacitorRecommendations');
   }
 
-  tStart('reviewPackages');
   if (!project.isCapacitor && !project.isCordova) {
     // The project is not using Cordova or Capacitor
     webProject(project);
@@ -380,8 +371,6 @@ export async function getRecommendations(project: Project, context: ExtensionCon
 
   project.add(new Tip('Settings', '', TipType.Settings).setQueuedAction(settings));
   project.add(new Tip('Show Logs', '', TipType.Files).setQueuedAction(showLogs));
-
-  tEnd('reviewPackages');
 }
 
 async function showLogs(queueFunction: QueueFunction) {