From 8cd88ad880627fe7b9c1c2039cdb9d8dd04f4e4f Mon Sep 17 00:00:00 2001 From: johnservo Date: Wed, 26 Apr 2023 11:37:17 +0800 Subject: [PATCH 1/3] Mgirated S3 stuff to commons --- backend/data_access/S3.js | 200 ------------------ lib/s3FormController.js | 2 +- .../[entity_type_slug]/[id].js | 13 +- .../[entity_type_slug]/create.js | 2 +- pages/admin/settings.js | 2 +- pages/api/[entity_type_slug].js | 2 +- pages/api/[entity_type_slug]/[id].js | 2 +- pages/api/settings.js | 2 +- pages/api/settings/[slug].js | 2 +- pages/api/settings/index.js | 2 +- 10 files changed, 10 insertions(+), 219 deletions(-) delete mode 100644 backend/data_access/S3.js diff --git a/backend/data_access/S3.js b/backend/data_access/S3.js deleted file mode 100644 index d4b01ed5..00000000 --- a/backend/data_access/S3.js +++ /dev/null @@ -1,200 +0,0 @@ -import AWS from "aws-sdk"; -import { generateRandVals } from "@klaudsol/commons/lib/Math"; -import { slsFetch } from "@klaudsol/commons/lib/Client"; - -const S3_ACCESS_KEY_ID = - process.env.KS_S3_ACCESS_KEY_ID ?? - process.env.KS_AWS_ACCESS_KEY_ID ?? - process.env.AURORA_AWS_ACCESS_KEY_ID; -const S3_SECRET_ACCESS_KEY = - process.env.KS_S3_SECRET_ACCESS_KEY ?? - process.env.KS_AWS_SECRET_ACCESS_KEY ?? - process.env.AURORA_AWS_SECRET_ACCESS_KEY; -//TODO: Deprecate AURORA_AWS_REGION on release V3.0.0 -const REGION = - process.env.KS_S3_REGION ?? - process.env.KS_AWS_REGION ?? - process.env.AURORA_AWS_REGION ?? 'us-east-1'; -const S3_BUCKET = process.env.KS_S3_BUCKET; - -const s3Config = { - accessKeyId: S3_ACCESS_KEY_ID, - secretAccessKey: S3_SECRET_ACCESS_KEY, - region: REGION, - signatureVersion: "v4", -}; - -export const initializeS3 = () => { - const s3 = new AWS.S3(s3Config); - - return s3; -}; - -export const getS3Param = (file) => { - return { - Bucket: S3_BUCKET, - Key: file.originalname, - Body: file.buffer, - ContentType: file.mimetype, - ACL: "public-read", - }; -}; - -export const generatePresignedUrl = (file) => { - const s3 = initializeS3(); - - const params = { - Bucket: S3_BUCKET, - Key: file.key, - Expires: 60, - ContentType: file.type, - ACL: "public-read", - }; - - const url = s3.getSignedUrl("putObject", params); - - return { url, originalName: file.originalName }; -}; - -export const generatePresignedUrls = async (fileNames) => { - const presignedUrls = fileNames.map(generatePresignedUrl); - - return presignedUrls; -}; - -export const uploadFileToUrl = async (file, url) => { - const uploadParams = { - method: "PUT", - headers: { - "Content-Type": "multipart/form-data", - }, - body: file, - }; - - await slsFetch(url, uploadParams); -}; - -export const uploadFilesToUrl = async (files, urls) => { - const promises = await urls.map(async (item) => { - const file = files.find((file) => file.name === item.originalName); - await uploadFileToUrl(file, item.url); - }); - - await Promise.all(promises); -}; - -export const generateUniqueKey = async (key) => { - const randVal = await generateRandVals(4); - const formattedKey = `${randVal}_${key}`; - - return formattedKey; -}; - -export const formatS3Param = async (param) => { - const uniqueKey = await generateUniqueKey(param.Key); - const newParams = { ...param, Key: uniqueKey }; - - return newParams; -}; - -export const generateEntries = (resFromS3, files, body) => { - const newBody = { ...body }; - files.forEach((file, i) => (newBody[file.fieldname] = resFromS3[i].Key)); - - return newBody; -}; - -export const generateEntry = (resFromS3, file) => { - const newBody = { [file.fieldname]: resFromS3.key }; - return newBody; -}; - -export const generateResource = (resFromS3, file) => { - const newBody = { key: file.fieldname, value: resFromS3.key }; - return newBody; -}; - -export const uploadFileToBucket = async (param) => { - const s3 = initializeS3(); - const res = await s3.upload(param).promise(); - - return res; -}; - -export const generateS3KeyForDeletion = (key) => { - return { - Key: key, - }; -}; - -export const generateS3ParamsForDeletion = (keysRaw) => { - const keys = keysRaw.map(generateS3KeyForDeletion); - const params = { - Bucket: S3_BUCKET, - Delete: { - Objects: keys, - }, - }; - - return params; -}; - -export const generateS3ParamForDeletion = (keyRaw) => { - const { Key } = generateS3KeyForDeletion(keyRaw); - const params = { - Bucket: S3_BUCKET, - Key, - }; - - return params; -}; - -export const deleteObjectsFromBucket = async (params) => { - const s3 = initializeS3(); - const res = await s3.deleteObjects(params).promise(); - - return res; -}; - -export const deleteObjectFromBucket = async (params) => { - const s3 = initializeS3(); - const res = await s3.deleteObject(params).promise(); - - return res; -}; - -export const deleteFilesFromBucket = async (toDelete) => { - const paramsForDeletion = generateS3ParamsForDeletion(toDelete); - await deleteObjectsFromBucket(paramsForDeletion); -}; - -export const addFileToBucket = async (file) => { - const paramsRaw = getS3Param(file); - const param = await formatS3Param(paramsRaw); - const resFromS3 = await uploadFileToBucket(param); - - return resFromS3; -}; - -export const addFilesToBucket = async (files) => { - const promise = await files.map(addFileToBucket); - const resFromS3 = await Promise.all(promise); - - return resFromS3; -}; - -// This function should only do one thing, but it does two things: -// Getting the params for deletion, and updating the files -// Need to refactor this to separate the uploading of the file -// And getting the params for deletion -// We also need a function to update one file. This function updates -// multiple files -export const updateFilesFromBucket = async (file, body, toDelete) => { - if (toDelete?.length) { - const paramsForDeletion = generateS3ParamsForDeletion(toDelete); - await deleteObjectsFromBucket(paramsForDeletion); - } - const uploadedFile = await addFilesToBucket(file, body); - - return uploadedFile; -}; diff --git a/lib/s3FormController.js b/lib/s3FormController.js index a525ec55..02c44e8b 100644 --- a/lib/s3FormController.js +++ b/lib/s3FormController.js @@ -1,4 +1,4 @@ -import { generateUniqueKey } from "@/backend/data_access/S3"; +import { generateUniqueKey } from "@klaudsol/commons/lib/S3"; export const convertToFormData = (entry) => { const formData = new FormData(); diff --git a/pages/admin/content-manager/[entity_type_slug]/[id].js b/pages/admin/content-manager/[entity_type_slug]/[id].js index 9cb8de13..80b4cb05 100644 --- a/pages/admin/content-manager/[entity_type_slug]/[id].js +++ b/pages/admin/content-manager/[entity_type_slug]/[id].js @@ -25,7 +25,7 @@ import { Formik, Form, Field } from "formik"; import ContentManagerLayout from "components/layouts/ContentManagerLayout"; import { DEFAULT_SKELETON_ROW_COUNT, writeContents } from "lib/Constants"; import { getAllFiles, getNonFiles, getBody } from "@/lib/s3FormController"; -import { uploadFilesToUrl } from "@/backend/data_access/S3"; +import { uploadFilesToUrl } from "@klaudsol/commons/lib/S3"; import AdminRenderer from "@/components/renderers/admin/AdminRenderer"; import { redirectToManagerEntitySlug } from "@/components/klaudsolcms/routers/routersRedirect"; @@ -123,13 +123,6 @@ export default function Type({ cache }) { return initialValues; }; - const getFilesToDelete = (values) => { - const files = Object.keys(values).filter((value) => values[value] instanceof File); - const keys = files.map((file) => state.values[file].key); - - return keys; - }; - const formikParams = { innerRef: formRef, initialValues: getFormikInitialVals(), @@ -138,13 +131,11 @@ export default function Type({ cache }) { try { dispatch({ type: SAVING }); - const { files, data, fileNames } = await getBody(values); - const toDelete = getFilesToDelete(values); + const { data, fileNames, files } = await getBody(values); const entry = { ...data, fileNames, - toDelete, entity_type_slug, entity_id: id, }; diff --git a/pages/admin/content-manager/[entity_type_slug]/create.js b/pages/admin/content-manager/[entity_type_slug]/create.js index 89af3161..43a60a15 100644 --- a/pages/admin/content-manager/[entity_type_slug]/create.js +++ b/pages/admin/content-manager/[entity_type_slug]/create.js @@ -26,7 +26,7 @@ import { import { FaCheck } from "react-icons/fa"; import { DEFAULT_SKELETON_ROW_COUNT, writeContents } from "lib/Constants"; import { getAllFiles, getNonFiles, getBody } from "@/lib/s3FormController"; -import { uploadFilesToUrl } from "@/backend/data_access/S3"; +import { uploadFilesToUrl } from "@klaudsol/commons/lib/S3"; import { redirectToManagerEntitySlug } from "@/components/klaudsolcms/routers/routersRedirect"; import classname from "classnames"; import AppBackButton from "@/components/klaudsolcms/buttons/AppBackButton"; diff --git a/pages/admin/settings.js b/pages/admin/settings.js index 80233980..0b1c6712 100644 --- a/pages/admin/settings.js +++ b/pages/admin/settings.js @@ -23,7 +23,7 @@ import { convertToFormData, getAllFiles, getBody } from "@/lib/s3FormController" import { validImageTypes } from "@/lib/Constants"; import { readSettings, modifyLogo } from "@/lib/Constants"; import { useClientErrorHandler } from "@/components/hooks" -import { uploadFilesToUrl } from "@/backend/data_access/S3"; +import { uploadFilesToUrl } from "@klaudsol/commons/lib/S3"; export default function Settings({ cache }) { const formRef = useRef(); diff --git a/pages/api/[entity_type_slug].js b/pages/api/[entity_type_slug].js index c309f057..b299651a 100644 --- a/pages/api/[entity_type_slug].js +++ b/pages/api/[entity_type_slug].js @@ -31,7 +31,7 @@ import { OK, NOT_FOUND } from "@klaudsol/commons/lib/HttpStatuses"; import { resolveValue } from "@/components/EntityAttributeValue"; import { setCORSHeaders, handleRequests } from "@klaudsol/commons/lib/API"; import { createHash } from "@/lib/Hash"; -import { addFilesToBucket, generateEntries, generatePresignedUrls } from "@/backend/data_access/S3"; +import { addFilesToBucket, generateEntries, generatePresignedUrls } from "@klaudsol/commons/lib/S3"; import { transformQuery, sortData } from "@/components/Util"; import { assert, assertUserCan } from '@klaudsol/commons/lib/Permissions'; import { filterData } from '@/components/Util'; diff --git a/pages/api/[entity_type_slug]/[id].js b/pages/api/[entity_type_slug]/[id].js index dc0165dc..b5e87584 100644 --- a/pages/api/[entity_type_slug]/[id].js +++ b/pages/api/[entity_type_slug]/[id].js @@ -27,7 +27,7 @@ import Entity from "@/backend/models/core/Entity"; import { deleteFilesFromBucket, generatePresignedUrls, -} from "@/backend/data_access/S3"; +} from "@klaudsol/commons/lib/S3"; import { withSession } from "@klaudsol/commons/lib/Session"; import { OK, NOT_FOUND } from "@klaudsol/commons/lib/HttpStatuses"; import { resolveValue } from "@/components/EntityAttributeValue"; diff --git a/pages/api/settings.js b/pages/api/settings.js index 4426053b..6b00ad76 100644 --- a/pages/api/settings.js +++ b/pages/api/settings.js @@ -5,7 +5,7 @@ import { generatePresignedUrls, deleteFilesFromBucket } - from "@/backend/data_access/S3"; + from "@klaudsol/commons/lib/S3"; import { resourceValueTypes } from "@/components/cmsTypes"; import { setCORSHeaders, handleRequests } from '@klaudsol/commons/lib/API'; import { OK, NOT_FOUND, BAD_REQUEST } from "@klaudsol/commons/lib/HttpStatuses"; diff --git a/pages/api/settings/[slug].js b/pages/api/settings/[slug].js index 680a1512..34adc4fb 100644 --- a/pages/api/settings/[slug].js +++ b/pages/api/settings/[slug].js @@ -19,7 +19,7 @@ import { generateEntries, generateResource, addFileToBucket, -} from "@/backend/data_access/S3"; +} from "@klaudsol/commons/lib/S3"; import { TYPES_REGEX } from "@/components/renderers/validation/TypesRegex"; import { readSettings, writeSettings } from "@/lib/Constants"; diff --git a/pages/api/settings/index.js b/pages/api/settings/index.js index a7858576..eb476ccd 100644 --- a/pages/api/settings/index.js +++ b/pages/api/settings/index.js @@ -5,7 +5,7 @@ import { generatePresignedUrls, deleteFilesFromBucket } - from "@/backend/data_access/S3"; + from "@klaudsol/commons/lib/S3"; import { resourceValueTypes } from "@/components/cmsTypes"; import { setCORSHeaders, handleRequests } from '@klaudsol/commons/lib/API'; import { OK, NOT_FOUND, BAD_REQUEST } from "@klaudsol/commons/lib/HttpStatuses"; From ee00b3a62b13f92d1e1ac01c447487740d48f2e1 Mon Sep 17 00:00:00 2001 From: johnservo Date: Wed, 26 Apr 2023 11:39:43 +0800 Subject: [PATCH 2/3] Fixed some stuff --- pages/admin/content-manager/[entity_type_slug]/[id].js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pages/admin/content-manager/[entity_type_slug]/[id].js b/pages/admin/content-manager/[entity_type_slug]/[id].js index 80b4cb05..5bc59406 100644 --- a/pages/admin/content-manager/[entity_type_slug]/[id].js +++ b/pages/admin/content-manager/[entity_type_slug]/[id].js @@ -123,6 +123,13 @@ export default function Type({ cache }) { return initialValues; }; + const getFilesToDelete = (values) => { + const files = Object.keys(values).filter((value) => values[value] instanceof File); + const keys = files.map((file) => state.values[file].key); + + return keys; + }; + const formikParams = { innerRef: formRef, initialValues: getFormikInitialVals(), @@ -131,7 +138,8 @@ export default function Type({ cache }) { try { dispatch({ type: SAVING }); - const { data, fileNames, files } = await getBody(values); + const { files, data, fileNames } = await getBody(values); + const toDelete = getFilesToDelete(values); const entry = { ...data, From d24cf342384159dfd636e272ee9221558af0cdb0 Mon Sep 17 00:00:00 2001 From: johnservo Date: Wed, 26 Apr 2023 11:40:43 +0800 Subject: [PATCH 3/3] Added toDelete back --- pages/admin/content-manager/[entity_type_slug]/[id].js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pages/admin/content-manager/[entity_type_slug]/[id].js b/pages/admin/content-manager/[entity_type_slug]/[id].js index 5bc59406..36e40327 100644 --- a/pages/admin/content-manager/[entity_type_slug]/[id].js +++ b/pages/admin/content-manager/[entity_type_slug]/[id].js @@ -126,7 +126,7 @@ export default function Type({ cache }) { const getFilesToDelete = (values) => { const files = Object.keys(values).filter((value) => values[value] instanceof File); const keys = files.map((file) => state.values[file].key); - + return keys; }; @@ -144,6 +144,7 @@ export default function Type({ cache }) { const entry = { ...data, fileNames, + toDelete, entity_type_slug, entity_id: id, };