feat(cozy-clisk): Implementing selective file dowload and fixing metadata deduplication

LucsT · LucsT · commit ecb3e4ec6aa8 · 2023-03-17T11:16:15.000+01:00
diff --git a/packages/cozy-clisk/src/contentscript/ContentScript.js b/packages/cozy-clisk/src/contentscript/ContentScript.js
@@ -1,4 +1,9 @@
 // @ts-check
+<<<<<<< Updated upstream
+=======
+import get from 'lodash/get'
+import clone from 'lodash/clone'
+>>>>>>> Stashed changes
 import waitFor from 'p-wait-for'
 import Minilog from '@cozy/minilog'
 
@@ -527,7 +532,8 @@ export default class ContentScript {
    */
   async storeFromWorker(obj) {
     // @ts-ignore Aucune surcharge ne correspond à cet appel.
-    Object.assign(this.store, obj)
+    //Object.assign(this.store, obj)
+    this.store = clone(obj)
   }
 
   onlyIn(csType, method) {
diff --git a/packages/cozy-clisk/src/index.js b/packages/cozy-clisk/src/index.js
@@ -5,6 +5,7 @@ export { MessengerInterface } from './bridge/bridgeInterfaces'
 export { default as ContentScript } from './contentscript/ContentScript'
 export { default as addData } from './launcher/addData'
 export { default as hydrateAndFilter } from './launcher/hydrateAndFilter'
+export { default as getFileIfExists } from './launcher/getFileIfExists'
 export { default as saveFiles } from './launcher/saveFiles'
 export { default as saveBills } from './launcher/saveBills'
 export { default as saveIdentity } from './launcher/saveIdentity'
diff --git a/packages/cozy-clisk/src/launcher.js b/packages/cozy-clisk/src/launcher.js
@@ -1,5 +1,6 @@
 export { default as addData } from './launcher/addData'
 export { default as hydrateAndFilter } from './launcher/hydrateAndFilter'
+export { default as getFileIfExists } from './launcher/getFileIfExists'
 export { default as saveFiles } from './launcher/saveFiles'
 export { default as saveBills } from './launcher/saveBills'
 export { default as saveIdentity } from './launcher/saveIdentity'
diff --git a/packages/cozy-clisk/src/launcher/getFileIfExists.js b/packages/cozy-clisk/src/launcher/getFileIfExists.js
@@ -0,0 +1,128 @@
+import Minilog from '@cozy/minilog'
+import { Q } from 'cozy-client'
+import get from 'lodash/get'
+
+const log = Minilog('getFileIfExists')
+
+module.exports = getFileIfExists
+
+async function getFileIfExists({ client, entry, options, folderPath, slug }) {
+  const fileIdAttributes = options.fileIdAttributes
+  const sourceAccountIdentifier = options.sourceAccountIdentifier
+
+  const isReadyForFileMetadata =
+    fileIdAttributes && slug && sourceAccountIdentifier
+  if (isReadyForFileMetadata) {
+    log.debug('Detecting file with metadata')
+    const file = await getFileFromMetaData(
+      client,
+      entry,
+      fileIdAttributes,
+      sourceAccountIdentifier,
+      slug
+    )
+    if (!file) {
+      // no file with correct metadata, maybe the corresponding file already exist in the default
+      // path from a previous version of the connector
+      log.debug('Rolling back on detection by filename')
+      return getFileFromPath(client, entry, folderPath)
+    } else {
+      return file
+    }
+  } else {
+    log.debug('Rolling back on detection by filename')
+    return getFileFromPath(client, entry, folderPath)
+  }
+}
+
+async function getFileFromMetaData(
+  client,
+  entry,
+  fileIdAttributes,
+  sourceAccountIdentifier,
+  slug
+) {
+  log.debug(
+    `Checking existence of ${calculateFileKey(entry, fileIdAttributes)}`
+  )
+  const files = await client.queryAll(
+    Q('io.cozy.files')
+      .where({
+        metadata: {
+          fileIdAttributes: calculateFileKey(entry, fileIdAttributes)
+        },
+        trashed: false,
+        cozyMetadata: {
+          sourceAccountIdentifier,
+          createdByApp: slug
+        }
+      })
+      .indexFields([
+        'metadata.fileIdAttributes',
+        'trashed',
+        'cozyMetadata.sourceAccountIdentifier',
+        'cozyMetadata.createdByApp'
+      ])
+  )
+  if (files && files[0]) {
+    if (files.length > 1) {
+      log.warn(
+        `Found ${files.length} files corresponding to ${calculateFileKey(
+          entry,
+          fileIdAttributes
+        )}`
+      )
+    }
+    return files[0]
+  } else {
+    log.debug('File not found')
+    return false
+  }
+}
+
+async function getFileFromPath(client, entry, folderPath) {
+  try {
+    log.debug(`Checking existence of ${getFilePath({ entry, folderPath })}`)
+    const result = await client
+      .collection('io.cozy.files')
+      .statByPath(getFilePath({ entry, folderPath }))
+    return result.data
+  } catch (err) {
+    log.debug(err.message)
+    return false
+  }
+}
+
+function getFilePath({ file, entry, folderPath }) {
+  if (file) {
+    return folderPath + '/' + getAttribute(file, 'name')
+  } else if (entry) {
+    return folderPath + '/' + getFileName(entry)
+  }
+}
+
+function getAttribute(obj, attribute) {
+  return get(obj, `attributes.${attribute}`, get(obj, attribute))
+}
+
+function calculateFileKey(entry, fileIdAttributes) {
+  return fileIdAttributes
+    .sort()
+    .map(key => get(entry, key))
+    .join('####')
+}
+
+function getFileName(entry) {
+  let filename
+  if (entry.filename) {
+    filename = entry.filename
+  } else {
+    log.error('Could not get a file name for the entry')
+    return false
+  }
+  return sanitizeFileName(filename)
+}
+
+function sanitizeFileName(filename) {
+  return filename.replace(/^\.+$/, '').replace(/[/?<>\\:*|":]/g, '')
+}
diff --git a/packages/cozy-clisk/src/launcher/getFileIfExists.spec.js b/packages/cozy-clisk/src/launcher/getFileIfExists.spec.js
@@ -0,0 +1,129 @@
+import getFileIfExists from './getFileIfExists'
+
+describe('getFileIfExists', function () {
+  function setup() {
+    const statByPath = jest.fn()
+    const queryAll = jest.fn()
+    const client = {
+      queryAll,
+      collection: () => ({
+        statByPath
+      })
+    }
+    return {
+      statByPath,
+      queryAll,
+      client
+    }
+  }
+
+  it('should use metadata if fileIdAttributes, slug and sourceAccountIdentifier are present', async () => {
+    const { client, queryAll } = setup()
+    queryAll.mockResolvedValue([{ _id: 'abcd' }]) // Mocking the return with a simple file obj
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: {
+        fileIdAttributes: ['filename'],
+        sourceAccountIdentifier: 'Identifier'
+      },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    // Test that the Q request cotains our parameters
+    expect(queryAll).toHaveBeenCalledWith(
+      expect.objectContaining({
+        selector: expect.objectContaining({
+          cozyMetadata: expect.objectContaining({
+            createdByApp: 'slug',
+            sourceAccountIdentifier: 'Identifier'
+          }),
+          metadata: expect.objectContaining({ fileIdAttributes: 'filename' })
+        })
+      })
+    )
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+
+  it('should roll back to filename when with metadata return nothing', async () => {
+    const { client, queryAll, statByPath } = setup()
+    queryAll.mockResolvedValue(undefined) // Mocking an empty return
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: {
+        fileIdAttributes: ['filename'],
+        sourceAccountIdentifier: 'Identifier'
+      },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    expect(queryAll).toHaveBeenCalledWith(
+      expect.objectContaining({
+        selector: expect.objectContaining({
+          cozyMetadata: expect.objectContaining({
+            createdByApp: 'slug',
+            sourceAccountIdentifier: 'Identifier'
+          }),
+          metadata: expect.objectContaining({ fileIdAttributes: 'filename' })
+        })
+      })
+    )
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+
+  it('shoud use filename if sourceAccountIdentifier is missing', async () => {
+    const { statByPath, client } = setup()
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: { fileIdAttributes: ['filename'] },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+
+  it('shoud use filename if slug is missing', async () => {
+    const { statByPath, client } = setup()
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: {
+        fileIdAttributes: ['filename'],
+        sourceAccountIdentifier: 'Identifier'
+      },
+      folderPath: '/fakepath'
+    })
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+  it('shoud use filename if fileIdAttributes is missing', async () => {
+    const { statByPath, client } = setup()
+    statByPath.mockResolvedValue({ data: { _id: 'abcd' } })
+    const file = await getFileIfExists({
+      client,
+      entry: {
+        filename: 'filename'
+      },
+      options: { sourceAccountIdentifier: 'Identifier' },
+      folderPath: '/fakepath',
+      slug: 'slug'
+    })
+    expect(statByPath).toHaveBeenCalledWith('/fakepath/filename')
+    expect(file).toStrictEqual({ _id: 'abcd' })
+  })
+})
diff --git a/packages/cozy-clisk/src/launcher/saveBills.js b/packages/cozy-clisk/src/launcher/saveBills.js
@@ -2,7 +2,7 @@
 import hydrateAndFilter from './hydrateAndFilter'
 import addData from './addData'
 import Minilog from '@cozy/minilog'
-import _ from 'lodash'
+import _, { lowerCase } from 'lodash'
 const log = Minilog('saveBills')
 const DOCTYPE = 'io.cozy.bills'
 
@@ -137,16 +137,22 @@ function convertCurrency(currency) {
 }
 
 function checkRequiredAttributes(entries) {
+  log.warn('Checking AAAABttributes')
   for (let entry of entries) {
     for (let attr in requiredAttributes) {
       if (entry[attr] == null) {
         throw new Error(
           `saveBills: an entry is missing the required ${attr} attribute`
         )
       }
+      log.warn(attr)
       const checkFunction = requiredAttributes[attr]
       const isExpectedType = _(entry[attr])[checkFunction]()
       if (isExpectedType === false) {
+        log.warn(entry.date)
+        log.warn(typeof entry.date)
+        log.warn(_.isDate(entry.date))
+        log.warn(entry)
         throw new Error(
           `saveBills: an entry has a ${attr} which does not respect ${checkFunction}`
         )
diff --git a/packages/cozy-clisk/src/launcher/saveBills.spec.js b/packages/cozy-clisk/src/launcher/saveBills.spec.js
@@ -51,17 +51,22 @@ describe('saveBills', function () {
     )
   })
   it('should check dates in bills', async () => {
-    expect.assertions(1)
+    //expect.assertions(1)
     await expect(async () => {
       return saveBills(
         [
           {
             filename: 'filename',
             fileurl: 'fileurl',
             amount: 12,
-            date: 'nodate',
+            //            date: 'nodate',
+            date: Date.now(),
             vendor: 'vendor',
+<<<<<<< Updated upstream
             fileDocument: { _id: 'testfileid' }
+=======
+            //fileDocument: {_id: 'testfileid'}
+>>>>>>> Stashed changes
           }
         ],
         {
@@ -72,13 +77,19 @@ describe('saveBills', function () {
       'saveBills: an entry has a date which does not respect isDate'
     )
   })
+<<<<<<< Updated upstream
   it('should convert valid date strings in bills', async () => {
     const result = await saveBills(
+=======
+  it('should save a bills with required attribute', async () => {
+    saveBills(
+>>>>>>> Stashed changes
       [
         {
           filename: 'filename',
           fileurl: 'fileurl',
           amount: 12,
+<<<<<<< Updated upstream
           date: '2012-09-02',
           vendor: 'vendor',
           fileDocument: { _id: 'testfileid' }
@@ -100,5 +111,15 @@ describe('saveBills', function () {
         vendor: 'vendor'
       }
     ])
+=======
+          //            date: 'nodate',
+          date: Date.now(),
+          vendor: 'vendor',
+          fileDocument: {_id: 'testfileid'}
+        }
+      ]
+    )
+
+>>>>>>> Stashed changes
   })
 })
diff --git a/packages/cozy-clisk/src/launcher/saveFiles.js b/packages/cozy-clisk/src/launcher/saveFiles.js
