Improve publish and preview speed

Author: sirugh

src/utils/preview.js

@@ -2,49 +2,62 @@ import config from '@adobe/aio-lib-core-config'
 import Logger from '@adobe/aio-lib-core-logging'
 const aioLogger = Logger('commerce:preview.js')
 
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
 /**
- *
- * @param files
+ * Given an array of filenames, uses the hlx admin api to preview the content.
+ * This function accounts for the rate limiting imposed on the API by previewing
+ * 10 files every second.
+ * @param {Array} files the array of files to be previewed
+ * @returns {Promise<Array>} that resolves to the array of results
  */
 export async function previewContent (files) {
   const { org, repo } = config.get('commerce.github')
   if (!org || !repo) throw new Error('Missing Github Org and Repo')
   console.log('⏳ Previewing files, this may take some time...')
+
   const results = []
-  const rateLimit = 10 // 10 requests per second
-  const interval = 1000 / rateLimit // interval in milliseconds
+  const batchSize = 10
+  let i = 0
 
-  const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+  while (i < files.length) {
+    const chunk = files.slice(i, Math.min(i + batchSize, files.length))
+    const chunkPromises = chunk.map(file => previewFile(file))
+    await Promise.all(chunkPromises)
+    await delay(1000)
+    i += batchSize
+  }
 
-  for (let i = 0; i < files.length; i++) {
-    aioLogger.debug('ORIGIN:', files[i])
-    let { pathname } = new URL(files[i])
+  /**
+   * TODO refactor this and publish
+   * @param file
+   */
+  async function previewFile (file) {
+    let { pathname } = new URL(file)
     if (pathname.endsWith('/')) {
       pathname = pathname.replace(/\/$/, '/index')
     }
-    aioLogger.debug('DESTINATION:', `https://da.live/#/${org}/${repo}${pathname.replace(/\.[^.]+$/, '')}`)
     const url = new URL(`https://admin.hlx.page/preview/${org}/${repo}/main${pathname}`)
-    aioLogger.debug(`Previewing: ${url}`)
+
+    aioLogger.debug(`Previewing {
+      "origin": "${file}",
+      "destination": "https://da.live/edit#/${org}/${repo}${pathname.replace(/\.[^.]+$/, '')}"
+    }`)
 
     let result
-    // TODO: refactor this retry/delay logic to be better!
     try {
       const res = await fetch(url, { method: 'POST' })
       if (res.status !== 200) {
-        result = { source: files[i], status: 'failed', message: `Failed to preview https://da.live/#/${org}/${repo}${pathname.replace(/\.[^.]+$/, '')}` }
+        result = { source: file, status: 'failed', message: `Failed to preview https://da.live/edit#/${org}/${repo}${pathname.replace(/\.[^.]+$/, '')}` }
         aioLogger.debug(res)
       } else {
-        result = { source: files[i], status: 'success' }
+        result = { source: file, status: 'success' }
       }
     } catch (error) {
-      result = { source: files[i], status: 'error', message: error.message }
+      result = { source: file, status: 'error', message: error.message }
       aioLogger.debug(error)
     }
     results.push(result)
-
-    if (i < files.length - 1) {
-      await delay(interval) // wait for the interval before making the next request
-    }
   }
 
   const successes = results.filter(({ status }) => status === 'success')
@@ -66,42 +79,54 @@ const filesToPublish = [
   '/footer',
   '/mini-cart'
 ]
+
 /**
  * For DA Live Preview, we must publish some files. This must be done AFTER preview.
  */
 export async function publishContent () {
   const { org, repo } = config.get('commerce.github')
   if (!org || !repo) throw new Error('Missing Github Org and Repo')
   console.log('⏳ Publishing some necessary files...')
+
   const results = []
-  const rateLimit = 10 // 10 requests per second
-  const interval = 1000 / rateLimit // interval in milliseconds
+  const batchSize = 10
+  let i = 0
 
-  const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+  while (i < filesToPublish.length) {
+    const chunk = filesToPublish.slice(i, Math.min(i + batchSize, filesToPublish.length))
+    const chunkPromises = chunk.map(file => publishFile(file))
+    await Promise.all(chunkPromises)
+    await delay(1000)
+    i += batchSize
+  }
 
-  for (let i = 0; i < filesToPublish.length; i++) {
-    const pathname = filesToPublish[i]
+  /**
+   *
+   * @param file
+   * @param pathname
+   */
+  async function publishFile (pathname) {
     const url = new URL(`https://admin.hlx.page/live/${org}/${repo}/main${pathname}`)
-    aioLogger.debug(`Publishing at ${url}`)
+
+    aioLogger.debug(`Publishing {
+        "origin": "${pathname}",
+        "destination": "https://da.live/edit#/${org}/${repo}${pathname.replace(/\.[^.]+$/, '')}"
+      }`)
 
     let result
     try {
       const res = await fetch(url, { method: 'POST' })
       if (res.status !== 200) {
-        result = { source: filesToPublish[i], status: 'failed', message: `Failed to publish ${filesToPublish[i]}` }
+        result = { source: pathname, status: 'failed', message: `Failed to publish https://da.live/edit#/${org}/${repo}${pathname.replace(/\.[^.]+$/, '')}` }
         aioLogger.debug(res)
       } else {
-        result = { source: filesToPublish[i], status: 'success' }
+        result = { source: pathname, status: 'success' }
       }
     } catch (error) {
-      result = { source: filesToPublish[i], status: 'error', message: error.message }
+      result = { source: pathname, status: 'error', message: error.message }
       aioLogger.debug(error)
     }
     results.push(result)
-
-    if (i < filesToPublish.length - 1) {
-      await delay(interval) // wait for the interval before making the next request
-    }
   }
 
   const successes = results.filter(({ status }) => status === 'success')