🚀 Enhanced Stability & Maintainability | Test Suite Expansion + Core Refactoring

.env.example

@@ -1,7 +1,7 @@
 NODE_ENV=production
-APP_ENV=DEV
+NEXT_PUBLIC_APP_ENV=DEV
 # auth
-NEXTAUTH_URL=http://localhost:8080
+NEXTAUTH_URL=http://localhost:3000
 NEXTAUTH_SECRET=tZuoxOlhNxw7Q/r3DCiuVKZJ7op1dX/XTLYqG3rEurA=
 
 # ai 

.github/workflows/devs.yml

@@ -33,8 +33,9 @@ jobs:
       # 2) generate a full .env before starting anything:
       - name: Generate Env File
         run: |
-         cat <<EOF > .env
-         APP_ENV=TEST
+         cat <<EOF > .env 
+         NEXT_PUBLIC_APP_ENV=TEST
+         DCUP_PARSER=http://localhost:9000
          NEXTAUTH_URL=http://localhost:3000
          REDIS_DB_ADDRESS=localhost
          REDIS_DB_PORT=6379
@@ -43,6 +44,18 @@ jobs:
          OPENAI_KEY="my key"
          NEXTAUTH_SECRET=tZuoxOlhNxw7Q/r3DCiuVKZJ7op1dX/XTLYqG3rEurA=
          DRIZZLE_DATABASE_URL=postgres://postgres:root@localhost:5432/dcup
+         NODE_ENV=production
+         OPENAI_ENDPOINT=https://models.inference.ai.azure.com
+         OPENAI_EMBEDDINGS_MODEL=text-embedding-3-small
+         OPENAI_AGENT_MODEL=gpt-4o-mini
+         QDRANT_DB_URL=http://localhost:6333
+         NEXT_PUBLIC_GOOGLE_CLIENT_ID=
+         GOOGLE_CLIENT_SECRET=
+         NEXT_PUBLIC_GOOGLE_API_KEY=
+         AUTH_GITHUB_ID=
+         AUTH_GITHUB_SECRET=
+         API_SECRET=xOAsKzZlYeGqaLiZNv6esJ
+         API_ACCESS_SECRET=fafkldsajfkldsjfoajFDIOASJFOfdlfja
          EOF
 
       # 3) spin up your other services (redis, postgres, etc):

DataSource/Aws/setAwsConnection.ts

@@ -7,13 +7,12 @@ import { connectionConfig } from "../utils";
 export const setAWSConnection = async (formData: FormData) => {
   const config = connectionConfig.safeParse({
     connectionId: formData.get("connectionId"),
-    identifier: formData.get("connectionName"),
+    identifier: formData.get("identifier"),
     folderName: formData.get("folderName"),
     folderId: formData.get("folderId"),
     metadata: formData.get("metadata"),
     pageLimit: formData.get("pageLimit"),
-    documentLimit: formData.get("documentLimit"),
-    maxPages: formData.get("maxPages")
+    fileLimit: formData.get("fileLimit"),
   })
 
   if (!config.success) {
@@ -36,14 +35,14 @@ export const setAWSConnection = async (formData: FormData) => {
     isConfigSet: true,
     isSyncing: true,
     limitPages:config.data.pageLimit,
-    limitFiles: config.data.documentLimit,
+    limitFiles: config.data.fileLimit,
   }).where(eq(connections.id, config.data.connectionId))
 
   return {
     connectionId: config.data.connectionId,
     service: "AWS",
-    pageLimit: config.data.maxPages,
-    fileLimit: config.data.documentLimit,
+    pageLimit: config.data.pageLimit,
+    fileLimit: config.data.fileLimit,
     metadata: config.data.metadata,
     files: [],
     links: []

DataSource/DirectUpload/SetNewConfigDirect/SetNewConfigDirect.tsx

@@ -16,7 +16,7 @@ export const SetNewConfigDirect = () => {
   return (
     <Dialog open={open} onOpenChange={setOpen}>
       <DialogTrigger asChild>
-        <Button>Upload Files</Button>
+        <Button data-test={"btn-upload-files"}>Upload Files</Button>
       </DialogTrigger>
       <DialogContent className="sm:max-w-[425px]">
         <DialogHeader>

DataSource/DirectUpload/UpdateConfigDirect/UpdateConfigDirect.tsx

@@ -19,7 +19,7 @@ export const UpdateConfigDirect = ({ connection, status }: { connection: Connect
 
   return (<Dialog open={open} onOpenChange={setOpen} >
     <DialogTrigger asChild>
-      <Button size='sm' variant={connection.isConfigSet ? 'ghost' : 'default'} disabled={status === 'PROCESSING' || (!status && connection.isSyncing)}>
+      <Button data-test="btn-config" size='sm' variant={connection.isConfigSet ? 'ghost' : 'default'} disabled={status === 'PROCESSING' || (!status && connection.isSyncing)}>
         <Settings2 />
         Configure
       </Button>

DataSource/DirectUpload/setDirectUploadConnection.ts

@@ -6,7 +6,7 @@ import { z } from "zod";
 
 const directUploadConfig = z.object({
   userId: z.string().min(5),
-  uploadName: z.string().min(1, { message: "Upload Name is required" }),
+  identifier: z.string().min(2),
   metadata: z.string()
     .transform((str, ctx): string => {
       try {
@@ -20,7 +20,7 @@ const directUploadConfig = z.object({
         return z.NEVER
       }
     }),
-  pageLimit: z.string().transform((str, ctx): number | null => {
+  pageLimit: z.string().nullable().transform((str, ctx): number | null => {
     try {
       if (str) return parseInt(str)
       return null
@@ -29,7 +29,7 @@ const directUploadConfig = z.object({
       return z.NEVER
     }
   }),
-  documentLimit: z.string().nullable().transform((str, ctx): number | null => {
+  fileLimit: z.string().nullable().transform((str, ctx): number | null => {
     try {
       if (str) return parseInt(str)
       return null
@@ -38,14 +38,6 @@ const directUploadConfig = z.object({
       return z.NEVER
     }
   }),
-  maxPages: z.string().transform((str, ctx): number => {
-    try {
-      return parseInt(str)
-    } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
-      return z.NEVER
-    }
-  }),
   files: z.array(z.any().refine((file) => {
     return (
       file ||
@@ -61,14 +53,23 @@ const directUploadConfig = z.object({
 
 const updateDirectUploadConfig = z.object({
   connectionId: z.string().min(5),
-  metadata: z.string()
-    .transform((str, ctx): string => {
+  pageLimit: z.string().nullable().transform((str, ctx): number | null => {
+    try {
+      if (str) return parseInt(str)
+      return null
+    } catch (error) {
+      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
+      return z.NEVER
+    }
+  }),
+  identifier: z.string().min(2).optional(),
+  metadata: z.string().optional()
+    .transform((str, ctx): string | undefined => {
       try {
         if (str) {
           JSON.parse(str)
           return str
         }
-        return "{}"
       } catch (e) {
         ctx.addIssue({ code: 'custom', message: 'Invalid JSON' })
         return z.NEVER
@@ -84,27 +85,19 @@ const updateDirectUploadConfig = z.object({
       message: "Invalid File",
     })
   ),
-  maxPages: z.string().transform((str, ctx): number => {
-    try {
-      return parseInt(str)
-    } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
-      return z.NEVER
-    }
-  }),
   links: z.array(z.string().min(5)),
   removedFiles: z.array(z.string().min(5))
 })
 
 export const updateDirectUploadConnection = async (formData: FormData) => {
-
   const config = updateDirectUploadConfig.safeParse({
     connectionId: formData.get("connectionId"),
+    identifier: formData.get("identifier"),
     metadata: formData.get("metadata") || "{}",
     files: formData.getAll("files") || [],
     links: formData.getAll("links") || [],
     removedFiles: formData.getAll("removedFiles") || [],
-    maxPages: formData.get("maxPages")
+    pageLimit: formData.get("pageLimit")
   })
 
   if (!config.success) {
@@ -119,6 +112,13 @@ export const updateDirectUploadConnection = async (formData: FormData) => {
 
   const connectionChunksIds: { chunksIds: string[] }[] = [];
 
+  await databaseDrizzle
+    .update(connections)
+    .set({
+      metadata: config.data.metadata,
+      identifier: config.data.identifier
+    }).where(eq(connections.id, config.data.connectionId))
+
   for (const fileName of config.data.removedFiles) {
     const files = await databaseDrizzle
       .delete(processedFiles)
@@ -146,10 +146,10 @@ export const updateDirectUploadConnection = async (formData: FormData) => {
   return {
     connectionId: config.data.connectionId,
     service: "DIRECT_UPLOAD",
-    metadata: config.data.metadata,
+    metadata: config.data.metadata ?? "{}",
     files: await Promise.all(files),
     links: config.data.links,
-    pageLimit: config.data.maxPages,
+    pageLimit: config.data.pageLimit,
     fileLimit: null
   }
 }
@@ -158,11 +158,10 @@ export const setDirectUploadConnection = async (formData: FormData) => {
 
   const config = directUploadConfig.safeParse({
     userId: formData.get("userId"),
-    uploadName: formData.get("uploadName"),
+    identifier: formData.get("identifier"),
     metadata: formData.get("metadata") || "{}",
     pageLimit: formData.get("pageLimit"),
-    documentLimit: formData.get("documentLimit"),
-    maxPages: formData.get("maxPages"),
+    fileLimit: formData.get("fileLimit"),
     files: formData.getAll("files"),
     links: formData.getAll("links"),
   })
@@ -177,21 +176,21 @@ export const setDirectUploadConnection = async (formData: FormData) => {
     throw new Error(`Validation errors - ${errors}`)
   }
 
-  const { files, links, userId, uploadName, metadata, documentLimit, pageLimit } = config.data;
+  const { files, links, userId, identifier, metadata, fileLimit, pageLimit } = config.data;
 
   if (files.length === 0 && links.length === 0) {
     throw new Error('Please provide at least one file or link to proceed.')
   }
 
   const conn = await databaseDrizzle.insert(connections).values({
     userId: userId,
-    identifier: uploadName,
+    identifier: identifier,
     service: 'DIRECT_UPLOAD',
     metadata: metadata,
     isConfigSet: true,
     isSyncing: true,
     limitPages: pageLimit,
-    limitFiles: documentLimit,
+    limitFiles: fileLimit,
   }).returning({ id: connections.id })
 
   const allFiles = files.map(async (file) => ({
@@ -208,8 +207,8 @@ export const setDirectUploadConnection = async (formData: FormData) => {
     metadata: metadata,
     files: await Promise.all(allFiles),
     links: links,
-    pageLimit: config.data.maxPages,
-    fileLimit: documentLimit
+    pageLimit: config.data.pageLimit,
+    fileLimit: fileLimit
   }
 }
 

DataSource/Dropbox/setDropboxConnection.ts

@@ -7,13 +7,12 @@ import { connectionConfig } from "../utils";
 export const setDropboxConnection = async (formData: FormData) => {
   const config = connectionConfig.safeParse({
     connectionId: formData.get("connectionId"),
-    identifier: formData.get("connectionName"),
+    identifier: formData.get("identifier"),
     folderName: formData.get("folderName"),
     folderId: formData.get("folderId"),
     metadata: formData.get("metadata"),
     pageLimit: formData.get("pageLimit"),
-    documentLimit: formData.get("documentLimit"),
-    maxPages: formData.get("maxPages"),
+    fileLimit: formData.get("fileLimit"),
   })
 
   if (!config.success) {
@@ -35,15 +34,15 @@ export const setDropboxConnection = async (formData: FormData) => {
     metadata: config.data.metadata,
     isConfigSet: true,
     isSyncing: true,
-    limitFiles: config.data.documentLimit,
+    limitFiles: config.data.fileLimit,
     limitPages: config.data.pageLimit,
   }).where(eq(connections.id, config.data.connectionId))
 
   return {
     connectionId: config.data.connectionId,
     service: "DROPBOX",
-    pageLimit: config.data.maxPages,
-    fileLimit: config.data.documentLimit,
+    pageLimit: config.data.pageLimit,
+    fileLimit: config.data.fileLimit,
     metadata: config.data.metadata,
     files: [],
     links: []

DataSource/GoogleDrive/setGoogleDriveConnection.ts

@@ -6,13 +6,12 @@ import { connectionConfig } from "../utils";
 export const setGoogleDriveConnection = async (formData: FormData) => {
   const config = connectionConfig.safeParse({
     connectionId: formData.get("connectionId"),
-    identifier: formData.get("connectionName"),
+    identifier: formData.get("identifier"),
     folderName: formData.get("folderName"),
     folderId: formData.get("folderId"),
     metadata: formData.get("metadata"),
     pageLimit: formData.get("pageLimit"),
-    documentLimit: formData.get("documentLimit"),
-    maxPages: formData.get("maxPages")
+    fileLimit: formData.get("fileLimit"),
   })
 
   if (!config.success) {
@@ -33,16 +32,16 @@ export const setGoogleDriveConnection = async (formData: FormData) => {
     } : undefined,
     metadata: config.data.metadata,
     limitPages: config.data.pageLimit,
-    limitFiles: config.data.documentLimit,
+    limitFiles: config.data.fileLimit,
     isConfigSet: true,
     isSyncing: true,
   }).where(eq(connections.id, config.data.connectionId))
 
   return {
     connectionId: config.data.connectionId,
     service: "GOOGLE_DRIVE",
-    pageLimit: config.data.maxPages,
-    fileLimit: config.data.documentLimit,
+    pageLimit: config.data.pageLimit,
+    fileLimit: config.data.fileLimit,
     metadata: config.data.metadata,
     files: [],
     links: []

DataSource/index.tsx

@@ -18,7 +18,6 @@ export const DataSource = ({ connection, token, status }: { connection: Connecti
       </>
     case "DIRECT_UPLOAD":
       return <>
-        {connection.isConfigSet && <SyncConnection connection={connection} status={status} />}
         <UpdateConfigDirect connection={connection} status={status} />
         <DeleteConnection connection={connection} status={status} />
       </>

DataSource/utils.ts

@@ -30,15 +30,7 @@ export const connectionConfig = z.object({
       return z.NEVER
     }
   }),
-  maxPages: z.string().transform((str, ctx): number  => {
-    try {
-      return parseInt(str)
-    } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
-      return z.NEVER
-    }
-  }),
-  documentLimit: z.string().nullable().transform((str, ctx): number | null => {
+  fileLimit: z.string().nullable().transform((str, ctx): number | null => {
     try {
       if (str) return parseInt(str)
       return null