Release 2.4.0

.labrc.js

@@ -0,0 +1,17 @@
+// Ignore items in global scope added by the AWS SDK for JavaScript v3.
+const globalsAsArray = [
+  '__extends', '__assign', '__rest', '__decorate', '__param', '__esDecorate', 
+  '__runInitializers', '__propKey', '__setFunctionName', '__metadata', 
+  '__awaiter', '__generator', '__exportStar', '__createBinding', '__values', 
+  '__read', '__spread', '__spreadArrays', '__spreadArray', '__await', 
+  '__asyncGenerator', '__asyncDelegator', '__asyncValues', '__makeTemplateObject', 
+  '__importStar', '__importDefault', '__classPrivateFieldGet', '__classPrivateFieldSet', 
+  '__classPrivateFieldIn', '__addDisposableResource', '__disposeResources', 
+  '__rewriteRelativeImportExtension'
+]
+
+const globals = globalsAsArray.toString()
+
+module.exports = {
+  globals
+}

lib/functions/ffoi-process.js

@@ -8,8 +8,8 @@ module.exports.handler = async (event) => {
   const bucket = event.Records[0].s3.bucket.name
   const key = event.Records[0].s3.object.key
   const data = await s3.getObject({ Bucket: bucket, Key: key })
-
-  const file = await util.parseXml(data.Body)
+  const bodyContents = await data.Body.transformToString()
+  const file = await util.parseXml(bodyContents)
 
   // use pool and not client due to multiple database queries
   const pool = new Pool({ connectionString: process.env.LFW_DATA_DB_CONNECTION })

lib/functions/rloi-process.js

@@ -8,8 +8,8 @@ module.exports.handler = async (event) => {
   const bucket = event.Records[0].s3.bucket.name
   const key = event.Records[0].s3.object.key
   const data = await s3.getObject({ Bucket: bucket, Key: key })
-
-  const file = await util.parseXml(data.Body)
+  const bodyContents = await data.Body.transformToString()
+  const file = await util.parseXml(bodyContents)
 
   // use pool and not client due to multiple database queries
   const pool = new Pool({ connectionString: process.env.LFW_DATA_DB_CONNECTION, max: 3 })

lib/functions/station-process.js

@@ -9,8 +9,8 @@ module.exports.handler = async (event) => {
   const key = event.Records[0].s3.object.key
 
   const data = await s3.getObject({ Bucket: bucket, Key: key })
-
-  const stations = await util.parseCsv(data.Body.toString())
+  const bodyContents = await data.Body.transformToString()
+  const stations = await util.parseCsv(bodyContents)
 
   const pool = new Pool({ connectionString: process.env.LFW_DATA_DB_CONNECTION })
   await station.saveToDb(stations, pool)

lib/helpers/invoke-lambda.js

@@ -1,10 +1,13 @@
-const { Lambda } = require('aws-sdk')
+const { LambdaClient, InvokeCommand } = require('@aws-sdk/client-lambda')
 
-const lambda = new Lambda()
+const lambda = new LambdaClient({})
 
 module.exports = function invokeLambda (functionName, payload) {
-  return lambda.invokeAsync({
+  const command = new InvokeCommand({
     FunctionName: functionName,
-    InvokeArgs: JSON.stringify(payload)
-  }).promise()
+    Payload: Buffer.from(JSON.stringify(payload)),
+    InvocationType: 'Event'
+  })
+
+  return lambda.send(command)
 }

lib/helpers/s3.js

@@ -1,19 +1,33 @@
-const AWS = require('aws-sdk')
-const s3 = new AWS.S3()
+const { S3Client, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, ListObjectsV2Command } = require('@aws-sdk/client-s3')
+const { NodeHttpHandler } = require('@smithy/node-http-handler')
+
+const s3 = new S3Client({
+  requestHandler: new NodeHttpHandler({
+    httpsAgent: { maxSockets: process.env.LFW_DATA_S3_MAX_SOCKETS }
+  }),
+  region: process.env.LFW_DATA_S3_REGION
+})
 
 module.exports = {
   getObject (params) {
-    return s3.getObject(params).promise()
+    const command = new GetObjectCommand(params)
+    return s3.send(command)
   },
+
   putObject (params) {
     // Put object then set ACL to allow bucket-owner-full-control
     params.ACL = 'bucket-owner-full-control'
-    return s3.putObject(params).promise()
+    const command = new PutObjectCommand(params)
+    return s3.send(command)
   },
+
   deleteObject (params) {
-    return s3.deleteObject(params).promise()
+    const command = new DeleteObjectCommand(params)
+    return s3.send(command)
   },
+
   listObjects (params) {
-    return s3.listObjectsV2(params).promise()
+    const command = new ListObjectsV2Command(params)
+    return s3.send(command)
   }
 }

lib/models/rloi.js

@@ -115,7 +115,8 @@ module.exports = {
 
           try {
             const result = await fetchStation(s3, bucket, `rloi/${item.$.region}/${item.$.stationReference}/station.json`)
-            station = JSON.parse(result.Body)
+            const bodyContents = await result.Body.transformToString()
+            station = JSON.parse(bodyContents)
           } catch (err) {
             // the console log is commented out so as not to spam the cloudwatch lambda
             // logging, as the s3.getObject throws an error when it can't find the object, and there

lib/models/station.js

@@ -14,7 +14,8 @@ module.exports = {
 
     console.log(`${count} stations to load`)
 
-    await Promise.all(stations.map(async station => {
+    // Sequential S3 uploads to avoid EMFILE errors
+    for (const station of stations) {
       uploaded++
       try {
         await s3.putObject({
@@ -23,10 +24,10 @@ module.exports = {
           Key: `rloi/${station.Region}/${station.Telemetry_ID}/station.json`
         })
       } catch (err) {
-        console.log(`Failed to upload(${uploaded}/${count}): ${params.Key}`)
+        console.log(`Failed to upload(${uploaded}/${count}): rloi/${station.Region}/${station.Telemetry_ID}/station.json`)
         console.error(err)
       }
-    }))
+    }
     console.log('Stations processed')
   },
 
@@ -86,9 +87,10 @@ module.exports = {
     await pool.query('deleteStations')
 
     // batch up the database inserts as struggles with > 1500 records
-    const stationsFactor = Math.floor(dbStations.length / 500)
+    const batchSize = 500
+    const stationsFactor = Math.floor(dbStations.length / batchSize)
     for (let i = 0; i <= stationsFactor; i++) {
-      const batch = dbStations.slice(i * 500, (i * 500) + 500)
+      const batch = dbStations.slice(i * batchSize, (i * batchSize) + batchSize)
       await pool.query('insertStations', batch)
     }