chore: use aws-sdk to increase aws support in eks pods

.circleci/config.yml

@@ -10,8 +10,7 @@ partials:
     APP_SESSION_SECRET: localSecret
     APP_SMTP_HOST: mailhog
     APP_SMTP_PORT: 1025
-    APP_STORAGE_ENDPOINT: localhost
-    APP_STORAGE_PORT: 9000
+    APP_STORAGE_ENDPOINT: "http://localhost:9000"
     APP_STORAGE_SSL: false
     APP_STORAGE_ACCESS_KEY: 'AKIAIOSFODNN7EXAMPLE'
     APP_STORAGE_SECRET_KEY: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'

.env.development

@@ -13,8 +13,7 @@ APP_DB_URI="mongodb://root:password@127.0.0.1:27017"
 APP_SMTP_PORT=1025
 
 # storage settings
-APP_STORAGE_ENDPOINT=localhost
-APP_STORAGE_PORT=9000
+APP_STORAGE_ENDPOINT="http://localhost:9000"
 APP_STORAGE_SSL=false
 APP_STORAGE_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
 APP_STORAGE_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"

.env.test

@@ -14,8 +14,7 @@ APP_SECURE_COOKIE=false
 APP_VERBOSE=false
 
 # storage settings
-APP_STORAGE_ENDPOINT=localhost
-APP_STORAGE_PORT=9000
+APP_STORAGE_ENDPOINT="http://localhost:9000"
 APP_STORAGE_SSL=false
 APP_STORAGE_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
 APP_STORAGE_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"

devtools/releases/cdn.js

@@ -1,6 +1,7 @@
+const { createReadStream } = require('fs');
 const path = require('path');
+const s3 = require('@aws-sdk/client-s3');
 const glob = require('glob');
-const Minio = require('minio');
 
 const assetsDir = path.resolve(__dirname, '../../build/public');
 
@@ -38,16 +39,17 @@ const verifyConditions = (pluginConfig, context) => {
 };
 
 const publish = async (pluginConfig, context) => {
-    const { bucket, endPoint, port = 443, useSSL = true, region } = pluginConfig;
+    const { bucket, endPoint, useSSL = true, region } = pluginConfig;
     const { logger, nextRelease } = context;
     const { version } = nextRelease;
 
-    const client = new Minio.Client({
-        endPoint,
-        port,
-        useSSL,
-        accessKey: process.env.CDN_ACCESS_KEY,
-        secretKey: process.env.CDN_SECRET_KEY,
+    const client = new s3.S3Client({
+        endpoint: endPoint,
+        credentials: {
+            accessKeyId: process.env.CDN_ACCESS_KEY,
+            secretAccessKey: process.env.CDN_SECRET_KEY,
+        },
+        sslEnabled: useSSL,
         region,
     });
 
@@ -60,7 +62,14 @@ const publish = async (pluginConfig, context) => {
 
     await Promise.all(
         assets.map(filename =>
-            client.fPutObject(bucket, `${version}/${filename}`, path.join(assetsDir, filename), { version })
+            client.send(
+                new s3.PutObjectCommand({
+                    Bucket: bucket,
+                    Key: `${version}/${filename}`,
+                    Metadata: { version },
+                    Body: createReadStream(path.join(assetsDir, filename)),
+                })
+            )
         )
     );
 };

devtools/webpack/index.ts

@@ -110,7 +110,8 @@ const serverConfig: Configuration = {
         // provide a package.json on production
         isBuildIntentProduction &&
             new PackagePlugin({
-                additionalModules: ['source-map-support'],
+                // we require aws4 to authenticate with IAM role on the database
+                additionalModules: ['source-map-support', 'aws4'],
             }),
 
         // show progress bar when building for production with TTY

docs/releasing.md

@@ -118,7 +118,6 @@ This plugin has the following settings
 | Env    | CDN_SECRET_KEY | Secret key                                            |
 | Plugin | bucket         | Bucket name                                           |
 | Plugin | endPoint       | Object Storage end point                              |
-| Plugin | port           | Object storage port (default: 443)                    |
 | Plugin | useSSL         | Object Storage end point requires SSL (default: true) |
 | Plugin | region         | Object storage region                                 |
 
@@ -135,7 +134,7 @@ module.exports = {
                 // bucket name
                 bucket: 'starter-kit',
                 // endpoint
-                endPoint: 's3.amazonaws.com',
+                endPoint: 'https://s3.amazonaws.com',
                 // aws region
                 region: 'ap-southeast-1',
             },

docs/unit-testing.md

@@ -49,11 +49,10 @@ test('dummyTest', () => {
 ## loadFixtures
 
 After using `setupDatabase` you may chain it to load fixtures with `loadFixtures`.
-The fixture must be wrote in EJSON.
 
 ```typescript
 import { composeHandlers, setupDatabase, cleanDatabase, loadFixtures } from './helpers';
-import dummyFixtures from './dummy.fixture.json';
+import dummyFixtures from './dummy.ts';
 
 // it will first connect to the database, execute migration then load fictures as given
 beforeEach(composeHandlers(setupDatabase, loadFixtures(dummyFixtures)));
@@ -62,15 +61,25 @@ beforeEach(composeHandlers(setupDatabase, loadFixtures(dummyFixtures)));
 afterEach(cleanDatabase);
 ```
 
-## setupEmptyBucket
+## createBucket
+
+Create a bucket by using `createBucket`.
+Should be used together with `dropBucket`.
+
+```typescript
+import { createBucket } from './helpers';
+
+beforeEach(createBucket);
+```
+
+## dropBucket
 
-You may ensure there's a bucket matching your configuration with an empty content by using `setupEmptyBucket`.
+Drop a bucket by using `dropBucket`.
 
 ```typescript
-import { setupEmptyBucket } from './helpers';
+import { dropBucket } from './helpers';
 
-// it will ensure the bucket exist and drop anything already there
-beforeEach(setupEmptyBucket);
+afterEach(dropBucket);
 ```
 
 ## getApolloClient

package.json

@@ -35,6 +35,7 @@
         "prepare": "husky install"
     },
     "devDependencies": {
+        "@aws-sdk/types": "^3.127.0",
         "@babel/core": "^7.18.10",
         "@babel/eslint-parser": "^7.18.9",
         "@babel/plugin-proposal-class-properties": "^7.18.6",
@@ -165,6 +166,9 @@
         "@ant-design/icons": "^4.7.0",
         "@ant-design/pro-layout": "^6.38.18",
         "@apollo/client": "^3.6.9",
+        "@aws-sdk/client-s3": "^3.150.0",
+        "@aws-sdk/client-ses": "^3.150.0",
+        "@aws-sdk/s3-request-presigner": "^3.150.0",
         "@babel/runtime": "^7.18.9",
         "@bull-board/api": "^4.2.2",
         "@bull-board/express": "^4.2.2",
@@ -184,7 +188,10 @@
         "apollo-server-core": "^3.10.1",
         "apollo-server-express": "^3.10.1",
         "apollo-upload-client": "^17.0.0",
+        "aws-sdk": "^2.1197.0",
+        "aws4": "^1.11.0",
         "bcryptjs": "^2.4.3",
+        "bl": "^5.0.0",
         "bson": "^4.6.5",
         "bull": "^4.8.5",
         "chalk": "^4.1.2",
@@ -213,7 +220,6 @@
         "ipaddr.js": "^2.0.1",
         "jsonwebtoken": "^8.5.1",
         "lodash": "^4.17.21",
-        "minio": "^7.0.30",
         "mjml": "^4.13.0",
         "mjml-react": "^2.0.8",
         "mongodb": "^4.8.1",

src/__tests__/core/storage.ts

@@ -0,0 +1,34 @@
+import { createReadStream } from 'fs';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+import { ObjectId } from 'mongodb';
+import fetch from 'node-fetch';
+import { uploadFile, getUploadedFile, getSignedUrlOnUploadedFile } from '../../server/core/storage';
+import streamToBuffer from '../../server/utils/streamToBuffer';
+import { cleanDatabase, composeHandlers, setupDatabase, createBucket, dropBucket } from '../helpers';
+
+beforeEach(composeHandlers(setupDatabase, createBucket));
+
+afterEach(composeHandlers(cleanDatabase, dropBucket));
+
+const testFile = join(__dirname, 'img.png');
+
+test('Test function uploadFile() followed by getUploadedFile()', async () => {
+    const stream = createReadStream(testFile);
+    const response = await uploadFile('jest', 'test-01.png', stream);
+    expect(ObjectId.isValid(response._id)).toBe(true);
+    expect(response.filename).toEqual('test-01.png');
+    expect(response.displayName).toEqual('test-01.png');
+    expect(response.uploadedAt instanceof Date).toEqual(true);
+    expect(response.etag).toEqual('"d6869304047d8495f2465a7b963e10ec"');
+    expect(response.objectName).toEqual(`jest/${response._id.toHexString()}.png`);
+    const uploadedFileStream = await getUploadedFile(response);
+    const uploadedFile = await streamToBuffer(uploadedFileStream);
+    const originalFile = await readFile(testFile);
+    expect(uploadedFile).toEqual(originalFile);
+    const url = await getSignedUrlOnUploadedFile(response);
+    const fetchResponse = await fetch(url);
+    expect(fetchResponse.status).toEqual(200);
+    const fetchedFile = await fetchResponse.buffer();
+    expect(fetchedFile).toEqual(originalFile);
+});

src/__tests__/helpers/storage.ts

@@ -1,38 +1,35 @@
+import {
+    CreateBucketCommand,
+    DeleteBucketCommand,
+    DeleteObjectsCommand,
+    ListObjectsV2Command,
+} from '@aws-sdk/client-s3';
 import config from '../../server/core/config';
-import { minioClient } from '../../server/core/storage';
+import { client } from '../../server/core/storage';
 
 const { bucket } = config.storage;
 
 const emptyBucket = async (): Promise<void> => {
-    const stream = await minioClient.listObjectsV2(bucket, '', true);
-
-    return new Promise((resolve, reject) => {
-        const objectNames = [];
-
-        stream.on('data', object => {
-            objectNames.push(object.name);
-        });
-
-        stream.on('error', reject);
-
-        stream.on('end', async () => {
-            if (objectNames.length) {
-                await minioClient.removeObjects(bucket, objectNames);
-            }
-
-            resolve();
-        });
-    });
+    const listObjectsResponse = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
+    const objectNames = listObjectsResponse.Contents.map(object => object.Key).filter(Boolean);
+
+    if (objectNames.length) {
+        await client.send(
+            new DeleteObjectsCommand({
+                Bucket: bucket,
+                Delete: {
+                    Objects: objectNames.map(objectName => ({ Key: objectName })),
+                },
+            })
+        );
+    }
 };
 
-// eslint-disable-next-line import/prefer-default-export
-export const setupEmptyBucket = async (): Promise<void> => {
-    const alreadyExist = await minioClient.bucketExists(bucket);
-
-    if (alreadyExist) {
-        await emptyBucket();
-        await minioClient.removeBucket(bucket);
-    }
+export const createBucket = async (): Promise<void> => {
+    await client.send(new CreateBucketCommand({ Bucket: bucket }));
+};
 
-    await minioClient.makeBucket(bucket, config.storage.provider.region);
+export const dropBucket = async (): Promise<void> => {
+    await emptyBucket();
+    await client.send(new DeleteBucketCommand({ Bucket: bucket }));
 };

src/server/core/config.ts

@@ -1,28 +1,64 @@
 import chalk from 'chalk';
-import type { SMTPSettings } from '../emails';
 import { getClientSideFieldLevelEncryptionSettings } from './encryption';
 import { getString, getBoolean, getInteger, getNumber, getPrefix, getStringList } from './env';
 
-const getSmtpSettings = (): SMTPSettings => {
-    const base = {
-        host: getString(getPrefix('SMTP_HOST'), 'localhost'),
-        port: getInteger(getPrefix('SMTP_PORT'), 465),
-        secure: getBoolean(getPrefix('SMTP_SECURE'), false),
-    };
+export type SMTPSettings = {
+    host: string;
+    port: number;
+    secure: boolean;
+    auth?: { user: string; pass: string };
+};
 
-    const user = getString(getPrefix('SMTP_USER'));
+export type AWSSESSettings = {
+    endpoint?: string;
+    region: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    sslEnabled: boolean;
+};
 
-    if (!user) {
-        return base;
+export type MailerSettings = ({ provider: 'smtp' } & SMTPSettings) | ({ provider: 'aws' } & AWSSESSettings);
+
+const getSmtpSettings = (): MailerSettings => {
+    const provider = getString(getPrefix('PROVIDER'), 'smtp');
+
+    switch (provider) {
+        case 'smtp': {
+            const base: MailerSettings = {
+                provider: 'smtp',
+                host: getString(getPrefix('SMTP_HOST'), 'localhost'),
+                port: getInteger(getPrefix('SMTP_PORT'), 465),
+                secure: getBoolean(getPrefix('SMTP_SECURE'), false),
+            };
+
+            const user = getString(getPrefix('SMTP_USER'));
+
+            if (!user) {
+                return base;
+            }
+
+            return {
+                ...base,
+                auth: {
+                    user,
+                    pass: getString(getPrefix('SMTP_PASSWORD')),
+                },
+            };
+        }
+
+        case 'aws':
+            return {
+                provider: 'aws',
+                endpoint: getString(getPrefix('SMTP_ENDPOINT')),
+                accessKeyId: getString(getPrefix('SMTP_ACCESS_KEY')),
+                secretAccessKey: getString(getPrefix('SMTP_SECRET_KEY')),
+                sslEnabled: getBoolean(getPrefix('SMTP_SSL'), true),
+                region: getString(getPrefix('SMTP_REGION'), 'ap-southeast-1'),
+            };
+
+        default:
+            throw new Error('SMTP provider not supported');
     }
-
-    return {
-        ...base,
-        auth: {
-            user,
-            pass: getString(getPrefix('SMTP_PASSWORD')),
-        },
-    };
 };
 
 const version = getString('VERSION', '0.0.0-development');
@@ -104,12 +140,13 @@ const config = {
 
     storage: {
         provider: {
-            endPoint: getString(getPrefix('STORAGE_ENDPOINT')),
-            accessKey: getString(getPrefix('STORAGE_ACCESS_KEY')),
-            secretKey: getString(getPrefix('STORAGE_SECRET_KEY')),
-            useSSL: getBoolean(getPrefix('STORAGE_SSL'), true),
-            port: getInteger(getPrefix('STORAGE_PORT')),
-            region: getString(getPrefix('STORAGE_REGION'), 'ap-southeast-1'),
+            endpoint: getString(getPrefix('STORAGE_ENDPOINT')),
+            credentials: {
+                accessKeyId: getString(getPrefix('STORAGE_ACCESS_KEY')),
+                secretAccessKey: getString(getPrefix('STORAGE_SECRET_KEY')),
+            },
+            sslEnabled: getBoolean(getPrefix('STORAGE_SSL'), true),
+            region: getString(getPrefix('STORAGE_REGION')),
         },
         bucket: getString(getPrefix('STORAGE_BUCKET'), 'app'),
     },