Add Tests

.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+Dockerfile
+.dockerignore
+coverage
+reports
+.DS_Store

.github/workflows/main.yml

@@ -0,0 +1,25 @@
+
+name: Run Synthetic tests
+
+on:
+  push:
+    branches: [ "*" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout 
+        uses: actions/checkout@v2
+      - name: run one version test
+        uses: sudo-bot/action-docker-compose@latest
+        with:
+          cli-args: -f ./regression-run/docker-compose.yml up --build s3fs-tests --exit-code-from s3fs-tests
+      - name: run multiversion test
+        uses: sudo-bot/action-docker-compose@latest
+        with:
+          cli-args: -f ./regression-run/docker-compose.yml up --build s3fs-test-multiple-versions --exit-code-from s3fs-test-multiple-versions
+

.gitignore

@@ -1 +1,5 @@
 .idea
+coverage
+node_modules
+.DS_Store
+temp

.npmignore

@@ -7,4 +7,5 @@ test
 .gitignore
 .jscsrc
 .jsinspectrc
-.idea
+.idea
+temp

README.md

@@ -1,14 +1,7 @@
 # S3FS
-[![npm](https://img.shields.io/npm/v/s3fs.svg)](https://www.npmjs.com/package/s3fs)
-[![npm](https://img.shields.io/npm/dm/s3fs.svg)](https://www.npmjs.com/package/s3fs)
-[![Build Status](https://travis-ci.org/RiptideElements/s3fs.svg?branch=master)](https://travis-ci.org/RiptideElements/s3fs)
-[![Coverage Status](https://img.shields.io/coveralls/RiptideElements/s3fs.svg)](https://coveralls.io/r/RiptideElements/s3fs)
-[![Codacy](https://img.shields.io/codacy/13e0385fd6fc4929a2d1a974c7d0d67f.svg)](https://www.codacy.com/public/davidtpate/s3fs)
-[![Code Climate](https://codeclimate.com/github/RiptideElements/s3fs/badges/gpa.svg)](https://codeclimate.com/github/RiptideElements/s3fs)
-[![David](https://img.shields.io/david/RiptideElements/s3fs.svg)](https://david-dm.org/RiptideElements/s3fs)
-[![David](https://img.shields.io/david/dev/RiptideElements/s3fs.svg)](https://david-dm.org/RiptideElements/s3fs)
-[![David](https://img.shields.io/david/peer/RiptideElements/s3fs.svg)](https://david-dm.org/RiptideElements/s3fs)
-
+```
+npm -i @hasnat/s3fs
+```
 Implementation of Node.JS [FS interface](http://nodejs.org/api/fs.html) using [Amazon Simple Storage Service (S3)](http://aws.amazon.com/s3/) for storage.
 
 **Lead Maintainer**: [David Pate](https://github.com/DavidTPate)
@@ -344,6 +337,18 @@ Lines        : 78.07% ( 356/456 )
 Additionally, an interactive HTML report will be generated in `./coverage/lcov-report/index.html` which allows browsing the coverage by file.
 
 
+## Fake s3 + Node versions tests using docker-compose
+This repository uses docker-compose to run tests on multiple node versions, using fake-s3 from localstack
+
+```bash
+docker-compose up --build -d s3fs-fake-s3 
+npm run test
+```
+or run via docker-compose
+```
+docker-compose -f regression-run/docker-compose.yml up s3fs-tests --no-recreate --exit-code-from s3fs-tests
+```
+
 ## License
 [MIT](LICENSE)
 

lib/s3fs.js

@@ -22,7 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-(function (module, fsInterface, util, Promise, AWS, s3Utils, extend, S3WriteStream, Stats) {
+(function (module, fsInterface, util, Promise, S3, s3Utils, extend, S3WriteStream, Stats) {
     var directoryRegExp = /\/$/,
         defaultCreateOptions = {};
 
@@ -40,7 +40,7 @@
         }
 
         var bucketParts = s3Utils.decomposePath(bucket);
-        this.s3 = options instanceof AWS.S3 ? options : new AWS.S3(options);
+        this.s3 = options instanceof S3 ? options : new S3(options);
         this.bucket = bucketParts[0];
         this.path = bucketParts.slice(1).join('/');
         if (this.path) {
@@ -78,21 +78,12 @@
     S3fs.prototype.exists = function (name, callback) {
         var self = this;
         var promise = new Promise(function (resolve) {
-            var key = s3Utils.toKey(s3Utils.joinPaths(self.path + s3Utils.toKey(name)));
-
-            if (directoryRegExp.test(key)) {
-                return resolve(true); //TODO: Can we query S3 for this to see if the directory actually exists?
-            }
-            self.s3.headObject(
-                {
-                    Bucket: self.bucket,
-                    Key: key
-                },
+            var prefix = s3Utils.toKey(s3Utils.joinPaths(self.path + s3Utils.toKey(name)));
+            if (prefix === "") prefix = "/";
+            self.s3.listObjectsV2(
+                {Bucket: self.bucket, Prefix: prefix, MaxKeys: 1},
                 function (err) {
-                    if (err) {
-                        return resolve(false);
-                    }
-                    resolve(true);
+                    err ? resolve(false): resolve(true);
                 });
         });
 
@@ -992,5 +983,5 @@
 
     module.exports = S3fs;
 
-}(module, require('./fsInterface'), require('util'), require('bluebird'), require('aws-sdk'), require('./utils'), require('extend'), require('./s3WriteStream'),
+}(module, require('./fsInterface'), require('util'), require('bluebird'), require('aws-sdk/clients/s3'), require('./utils'), require('extend'), require('./s3WriteStream'),
     require('./Stats')));

package.json

@@ -1,6 +1,6 @@
 {
   "dependencies": {
-    "aws-sdk": "~2.814.0",
+    "aws-sdk": "~2.1333.0",
     "bluebird": "~3.4.1",
     "extend": "~3.x"
   },
@@ -41,7 +41,8 @@
     "lint": "eslint . && jscs . && jsinspect . && buddy --detect-objects index.js ./lib",
     "security": "nsp check",
     "test": "mocha --check-leaks --recursive --timeout=25000 test",
+    "test-regression": "node regression-run/test-multiple-versions/regression.js",
     "test-ci": "npm run lint && npm run security && npm run cover"
   },
-  "version": "2.5.3"
+  "version": "2.5.4"
 }

regression-run/Dockerfile

@@ -0,0 +1,13 @@
+ARG NODEJS_VERSION
+
+FROM node:${NODEJS_VERSION}
+
+ARG NODE_ENV=development
+ENV NODE_ENV=${NODE_ENV}
+WORKDIR /usr/local/src/s3fs
+
+COPY package.json .
+RUN npm install
+COPY . .
+
+CMD ["npm", "run", "test"]

regression-run/README.md

@@ -0,0 +1,16 @@
+# S3FS Tests
+
+Docker Compose is used to build and deploy a test environment.
+
+test.sh waits on tests exit code.
+## Run tests
+```
+docker-compose -f regression-run/docker-compose.yml up -d --no-recreate fake-s3
+docker-compose -f regression-run/docker-compose.yml up --build s3fs-tests --no-recreate --exit-code-from s3fs-tests
+docker-compose -f regression-run/docker-compose.yml up --build s3fs-test-multiple-versions --no-recreate --exit-code-from s3fs-test-multiple-versions
+```
+
+## Run multiple versions test
+```
+./regression-run/test-multiple-versions.sh
+```

regression-run/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3.7'
+
+services:
+
+  s3fs-fake-s3:
+    build: fake-s3
+    ports:
+      - "4566:4566"
+#      - "8080:8080"
+    environment:
+      - SERVICES=s3
+      - PORT_WEB_UI=8080
+      - HOST_TMP_FOLDER=/tmp/localstack
+    tmpfs:
+      - /tmp/localstack:exec,mode=600
+    healthcheck:
+      test: awslocal s3api wait bucket-exists --bucket bucket-a && awslocal s3api wait bucket-exists --bucket bucket-b
+
+
+  s3fs-tests:
+    container_name: "s3fs-tests-node-${NODEJS_VERSION:-16.3.0}"
+    build:
+      context: ..
+      dockerfile: ./regression-run/Dockerfile
+      args:
+        NODEJS_VERSION: "${NODEJS_VERSION:-16.3.0}"
+        NODE_ENV: "${NODE_ENV:-development}"
+    depends_on:
+      s3fs-fake-s3:
+        condition: service_healthy
+    environment:
+      AWS_ACCESS_KEY_ID: 'access-key-dont-matter-for-fake-s3'
+      AWS_SECRET_KEY: 'secret-key-dont-matter-for-fake-s3'
+      AWS_ENDPOINT: 'http://s3fs-fake-s3:4566/'
+    # defaulting command to unit tests, for multiversion we will use slimmer regression.js
+    command: "${TEST_COMMAND:-npm run test}"
+
+
+  s3fs-test-multiple-versions:
+    build:
+      context: ..
+      dockerfile: ./regression-run/test-multiple-versions/Dockerfile
+    environment:
+      NODEJS_VERSIONS_TO_TEST_UNIT: "${NODEJS_VERSIONS_TO_TEST_UNIT:-19.7.0 18.15.0 17.9.1 16.19.1 15.14.0 14.21.3 13.14.0 12.22.12 11.15.0 10.24.1 9.11.2 8.17.0 7.10.1 6.17.1}"
+      NODEJS_VERSIONS_TO_TEST_REGRESSION: "${NODEJS_VERSIONS_TO_TEST_REGRESSION:-19.7.0 18.15.0 17.9.1 16.19.1 15.14.0 14.21.3 13.14.0 12.22.12 11.15.0 10.24.1 9.11.2 8.17.0 7.10.1}"
+      NODEJS_VERSIONS_TO_TEST_UNTESTED: "${NODEJS_VERSIONS_TO_TEST_UNTESTED:-6.17.1 5.12.0 4.9.1 0.12.18}"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"

regression-run/fake-s3/Dockerfile

@@ -0,0 +1,4 @@
+FROM localstack/localstack:0.11.5
+
+COPY s3 /local-s3-data/
+COPY import-s3-buckets.sh /docker-entrypoint-initaws.d/import-s3-buckets.sh

regression-run/fake-s3/README.md

@@ -0,0 +1,5 @@
+# A Fake S3 for testing
+
+A Fake S3 service for testing,
+
+Data from s3 directory is copied to s3 service on boot.

regression-run/fake-s3/import-s3-buckets.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+cd /local-s3-data
+
+for BUCKET in `find . -maxdepth 1 -type d | sed 's#\.\/##'`
+do
+  if [[ "$BUCKET" != "." ]]
+  then
+    (awslocal s3 mb s3://${BUCKET} || true)
+    awslocal s3 sync ./${BUCKET}/ s3://${BUCKET} --exclude "*.DS_Store*"
+  fi
+done

regression-run/fake-s3/s3/bucket-a/bucket-a-item-1.txt

@@ -0,0 +1 @@
+bucket-a-item-2

regression-run/fake-s3/s3/bucket-a/bucket-a-item-2.txt

@@ -0,0 +1 @@
+bucket-a-item-1

regression-run/fake-s3/s3/bucket-b/directory/bucket-b-item-1.txt

@@ -0,0 +1 @@
+bucket-b-item-1

regression-run/fake-s3/s3/bucket-b/directory/nested/bucket-b-item-2.txt

@@ -0,0 +1 @@
+bucket-b-item-2

regression-run/test-multiple-versions/Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine:3.16
+
+WORKDIR /usr/local/src/s3fs
+COPY --from=docker /usr/libexec/docker/cli-plugins/docker-compose /usr/bin/docker-compose
+COPY ./regression-run/test-multiple-versions/entrypoint.sh /entrypoint.sh
+COPY . .
+
+ENTRYPOINT /entrypoint.sh
+

regression-run/test-multiple-versions/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env sh
+set -euxo pipefail
+
+function cleanup()
+{
+    exit_status=$?
+    echo "exit was $exit_status"
+
+    docker-compose -f regression-run/docker-compose.yml rm s3fs-fake-s3 -f  || true
+
+    exit "$exit_status"
+}
+trap cleanup EXIT
+
+docker-compose -f regression-run/docker-compose.yml up --build -d --no-recreate s3fs-fake-s3
+
+for NODEJS_VERSION in $NODEJS_VERSIONS_TO_TEST_UNIT
+do
+    echo "======================Start unit testing nodejs version ${NODEJS_VERSION}============================="
+    NODEJS_VERSION="${NODEJS_VERSION}" \
+    NODE_ENV=development \
+    TEST_COMMAND="npm run test" \
+    docker-compose -f regression-run/docker-compose.yml up --build s3fs-tests --exit-code-from s3fs-tests
+done
+
+
+for NODEJS_VERSION in $NODEJS_VERSIONS_TO_TEST_REGRESSION
+do
+    echo "======================Start regression testing nodejs version ${NODEJS_VERSION}============================="
+    NODEJS_VERSION="${NODEJS_VERSION}" \
+    NODE_ENV=production \
+    TEST_COMMAND="npm run test-regression" \
+    docker-compose -f regression-run/docker-compose.yml up --build s3fs-tests --exit-code-from s3fs-tests
+done

regression-run/test-multiple-versions/regression.js

@@ -0,0 +1,33 @@
+function decorate(func, param1, param2) {
+    const f = func()
+    return (function run(input) { f.next(input).then(run) })()
+}
+const _main = (async function () {
+    'use strict';
+    try {
+        var S3FS = require('../../index'), s3Credentials = require('../../test/_s3Credentials')
+
+        console.log('Regression Testing')
+
+        var bucketName = 's3fs-bucket-test-bucket-' + (Math.random() + '').slice(2, 8),
+            s3fsImpl = new S3FS(bucketName, s3Credentials);
+
+
+        var created = await s3fsImpl.create()
+
+        console.log('bucket creation response', created)
+
+        console.log('file creation response', (await s3fsImpl.writeFile('file-1.json', '{ "test": "test" }')))
+        console.log('ls', (await s3fsImpl.readdir('/')))
+        if ((await s3fsImpl.readdir('/')).length === 1) {
+            return process.exit(0);
+        }
+        process.exit(1)
+    } catch (e) {
+        console.log('error', e)
+        process.exit(1)
+    }
+})
+
+
+_main();

test/_s3Credentials.js

@@ -0,0 +1,24 @@
+if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_KEY) {
+        // for local debugging
+        process.env.AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || 'AKIAIOSFODNN7EXAMPLE';
+        process.env.AWS_SECRET_KEY = process.env.AWS_SECRET_KEY || 'wJalrXUtnFEMI/K7MDENG/bZCI/K';
+        process.env.AWS_ENDPOINT = process.env.AWS_ENDPOINT || 'http://127.0.0.1:4566/';
+        console.log(`Using fake s3 credentials targeting s3 endpoint ${process.env.AWS_ENDPOINT}`);
+}
+
+
+// console.log(process.env)
+module.exports = {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_KEY,
+
+    // used for fake-s3... for aws sdk
+    // to use http://fake-s3:4572/bucket-name  (ok host in docker network)
+    // not do http://bucket-name.fake-s3:4572/ (unresolved host in docker network)
+        hostPrefixEnabled: false,
+        s3ForcePathStyle: true,
+        forcePathStyle: true,
+        pathStyleAccessEnabled: true,
+        // s3BucketEndpoint: true,
+        endpoint: process.env.AWS_ENDPOINT
+    };