Add an S3 backend

.gitignore

@@ -103,4 +103,5 @@ dist
 # TernJS port file
 .tern-port
 
-data
+data
+ocfl/types

README.md

@@ -112,3 +112,26 @@ List all existing files in the object
         let fileContent = await object.getFile(f).asString();
     }
 
+## Development
+
+Setup the development environment
+
+```bash
+npm install --workspaces
+
+# Set up the fixtures (only do this once)
+git submodule init
+git submodule update
+```
+
+NOTE: You only need to run ```npm install``` at the root ofthe git respository as we are using workspaces.
+
+### Tests
+
+```bash
+# To run tests for all packages
+npm test
+
+# To run tests for one package
+cd ocfl-fs; npm test
+```

ocfl-fs/README.md

@@ -1,5 +1,5 @@
 # ocfl-fs - OCFL implementation using Filesystem storage backend for Node.js
-This is JavaScript/Node.js library to create and interact with Oxford Common File Layout (OCFL) storage and objects within it. 
+This is JavaScript/Node.js library to create and interact with Oxford Common File Layout (OCFL) storage and objects within it.
 This package implements the OCFL client that stores storage root and its objects on a locally attached filesystem.
 
 ## Installation
@@ -10,13 +10,17 @@ This package implements the OCFL client that stores storage root and its objects
 
 The storage and object methods accept an optional fs parameter, which can be any module that implements Node.js fs module API. By default, it will use the built in fs module.
 
+```js
     const ocfl = require('@ocfl/ocfl-fs');
     const storage = ocfl.storage({root: '/var/data/myocfl'});
+```
 
-For example, by passing it memfs (https://www.npmjs.com/package/memfs) the OCFL storage will be created in memory using the memfs module: 
+For example, by passing it memfs (https://www.npmjs.com/package/memfs) the OCFL storage will be created in memory using the memfs module:
 
+```js
     const ocfl = require('@ocfl/ocfl-fs');
     const fs = require('memfs');
     const storage = ocfl.storage({root: '/var/data/myocfl'}, {fs});
+```
 
-For common usage documentation please refer to the [README.md in GitHub](https://github.com/Arkisto-Platform/ocfl-js)
+For common usage documentation please refer to the [README.md in GitHub](https://github.com/Language-Research-Technology/ocfl-js)

ocfl-fs/index.mjs

@@ -1,3 +1,3 @@
-export * from './index.js'; 
+export * from './index.js';
 import ocfl from './index.js';
 export default ocfl;

ocfl-fs/lib/store.js

@@ -37,8 +37,17 @@ class OcflFsStore extends OcflStore {
     this.fs = config?.fs ?? fs;
   }
 
-  async stat(filePath) { 
-    return this.fs.promises.stat(filePath);    
+  /**
+   * Check if file exists
+   * @param {string} filePath
+   */
+  async exists(filePath) {
+    try {
+      await this.fs.promises.stat(filePath);
+      return true;
+    } catch (error) {
+      return false;
+    }
   }
 
   async createReadStream(filePath, options) {
@@ -48,7 +57,10 @@ class OcflFsStore extends OcflStore {
 
   async createWriteStream(filePath, options) { 
     await this.fs.promises.mkdir(path.dirname(filePath), { recursive: true });
-    return this.fs.createWriteStream(filePath, options);
+    const ws = this.fs.createWriteStream(filePath, options);
+    const promise = Promise.resolve();
+
+    return { ws, promise };
   }
 
   async readFile(filePath, options) {
@@ -98,4 +110,4 @@ class OcflFsStore extends OcflStore {
 
 };
 
-module.exports = { OcflFsStore };
+module.exports = { OcflFsStore };

ocfl-fs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ocfl/ocfl-fs",
   "version": "0.2.0",
-  "description": "Oxford Common File Layout (OCFL) JS client library for filesystem storage",
+  "description": "Oxford Common File Layout (OCFL) JS client library for S3 storage",
   "main": "index.js",
   "exports": {
     ".": [
@@ -27,7 +27,7 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/Arkisto-Platform/ocfl-js"
+    "url": "https://github.com/Language-Research-Technology/ocfl-js"
   },
   "author": "Alvin Sebastian (https://orcid.org/0000-0002-9086-1722)",
   "contributors": [
@@ -40,8 +40,7 @@
     "clean-jsdoc-theme": "^3.3.4",
     "jsdoc": "^3.6.10",
     "mocha": "^10.0.0",
-    "typedoc": "^0.22.15",
-    "@ocfl/ocfl-tests": "*"
+    "typedoc": "^0.22.15"
   },
   "dependencies": {
     "@ocfl/ocfl": "0.2.0"

ocfl-fs/test/object.spec.js

@@ -3,13 +3,15 @@
 
 const ocfl = require('../index');
 
+const { storeConfig, helpers } = require('./setup');
+
 describe("OcflObject class using OcflFsStore", function () {
   // it("constructor", function() {
   //   console.log(this.ocfl);
   //   assert.strictEqual(DIGEST.size(), 2);
   //   assert.strictEqual(DIGEST_FIXITY.size(), 5);
   // });
 
-  require('@ocfl/ocfl-tests').object(ocfl);
+  require('@ocfl/ocfl-tests').object(ocfl, storeConfig, helpers);
 });
 

ocfl-fs/test/setup.js

@@ -0,0 +1,12 @@
+const fs = require('fs');
+
+const helpers = {
+  getFile: async (filePath) => fs.promises.readFile(filePath, 'utf8')
+};
+
+const storeConfig = { };
+
+module.exports = {
+  storeConfig,
+  helpers,
+};

ocfl-s3/.eslintrc.json

@@ -0,0 +1,37 @@
+{
+  "env": {
+    "node": true,
+    "es2022": true,
+    "mocha": true
+  },
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "airbnb-base"
+  ],
+  "overrides": [],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": "latest"
+  },
+  "plugins": [
+    "@typescript-eslint",
+    "mocha"
+  ],
+  "rules": {
+    "max-len": ["warn", { "code": 180 }],
+    "import/prefer-default-export": "off",
+    "no-unused-vars": "off",
+    "import/extensions": ["error", "ignorePackages", {
+      "js": "never",
+      "ts": "never"
+    }]
+  },
+  "settings": {
+    "import/resolver": {
+      "typescript": {
+        "alwaysTryTypes": true
+      }
+    }
+  }
+}

ocfl-s3/README.md

@@ -1,4 +1,18 @@
-# ocfl-js - Filesystem storage implementation
-This is JavaScript/Node.js library to create and interact with Oxford Common File Layout (OCFL) storage and objects within it. 
-This package implements the OCFL client that stores storage root and its objects on a locally attached filesystem.
+# ocfl-s3 - OCFL implementation using S3 storage backend for Node.js
+This is JavaScript/Node.js library to create and interact with Oxford Common File Layout (OCFL) storage and objects within it.
+This package implements the OCFL client that stores storage root and its objects in an S3 bucket.
 
+## Installation
+
+```bash
+    npm install @ocfl/ocfl-s3
+```
+
+## Usage
+
+```js
+    const ocfl = require('@ocfl/ocfl-fs');
+    const storage = ocfl.storage({ bucket: 'my-ofcl-bucket' });
+```
+
+For common usage documentation please refer to the [README.md in GitHub](https://github.com/Language-Research-Technology/ocfl-js)

ocfl-s3/jsdoc.json

@@ -1,6 +1,6 @@
 {
   "source": {
-    "include": ["lib"],
+    "include": ["src"],
     "exclude": ["node_modules"]
   },
   "plugins": ["plugins/markdown"],
@@ -17,4 +17,4 @@
       "theme": "light"
     }
   }
-}
+}

ocfl-s3/package.json

@@ -1,20 +1,18 @@
 {
-  "name": "ocfl-fs",
-  "version": "1.1.0",
+  "name": "@ocfl/ocfl-s3",
+  "version": "0.2.0",
   "description": "Oxford Common File Layout (OCFL) JS client library for filesystem storage",
-  "main": "index.js",
   "exports": {
-    ".": [
-      {
-        "import": "./index.mjs",
-        "require": "./index.js",
-        "default": "./index.js"
-      },
-      "./index.js"
-    ]
+    "require": "./dist/cjs/index.js",
+    "import": "./dist/esm/index.js",
+    "types": "./dist/types/index.d.ts",
+    "default": "./dist/esm/index.js"
   },
   "scripts": {
-    "test": "mocha",
+    "compile": "tsc -p ./tsconfig.cjs.json; tsc -p ./tsconfig.esm.json; tsc -p ./tsconfig.types.json",
+    "build:clean": "rm -rf ./dist",
+    "build": "npm run build:clean; npm run compile; ./scripts/prepare-package-json",
+    "test": "ts-mocha test/*.ts",
     "build-docs2": "typedoc --tsconfig tsconfig.json",
     "build-docs": "jsdoc -c jsdoc.json"
   },
@@ -23,11 +21,11 @@
     "Oxford Common File Layout",
     "preservation",
     "storage",
-    "filesystem"
+    "s3"
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/Arkisto-Platform/ocfl-js"
+    "url": "https://github.com/Language-Research-Technology/ocfl-js"
   },
   "author": "Alvin Sebastian (https://orcid.org/0000-0002-9086-1722)",
   "contributors": [
@@ -36,14 +34,36 @@
   ],
   "license": "GPL-3.0-or-later",
   "devDependencies": {
-    "@types/fs-extra": "^9.0.13",
-    "clean-jsdoc-theme": "^3.3.4",
-    "jsdoc": "^3.6.10",
+    "@aws-sdk/util-stream-node": "^3.341.0",
+    "@tsconfig/esm": "^1.0.3",
+    "@tsconfig/node18": "^2.0.1",
+    "@tsconfig/strictest": "^2.0.1",
+    "@types/node": "^20.2.5",
+    "@types/s3rver": "^3.7.0",
+    "@typescript-eslint/eslint-plugin": "^5.59.8",
+    "@typescript-eslint/parser": "^5.59.8",
+    "clean-jsdoc-theme": "^4.2.7",
+    "eslint": "^8.41.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-plugin-import": "^2.27.5",
+    "glob": "^10.2.6",
+    "install": "^0.13.0",
+    "jsdoc": "^4.0.2",
     "mocha": "^10.0.0",
-    "typedoc": "^0.22.15"
+    "npm": "^9.6.7",
+    "s3rver": "^3.7.1",
+    "typedoc": "^0.24.7",
+    "typescript": "^5.0.4"
   },
   "dependencies": {
-    "fs-extra": "^10.1.0",
-    "ocfl": "1.1.0"
-  }
+    "@aws-sdk/client-s3": "^3.341.0",
+    "@aws-sdk/lib-storage": "^3.341.0",
+    "@ocfl/ocfl": "0.2.0",
+    "ts-mocha": "^10.0.0"
+  },
+  "files": [
+    "dist/*",
+    "src/*",
+    "README.md"
+  ]
 }

ocfl-s3/scripts/prepare-package-json

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+cat > dist/cjs/package.json <<!EOF
+{
+  "type": "commonjs"
+}
+!EOF
+
+cat > dist/esm/package.json <<!EOF
+{
+  "type": "module"
+}
+!EOF

ocfl-s3/src/index.ts

@@ -0,0 +1,6 @@
+import { Ocfl } from '@ocfl/ocfl';
+
+import { OcflS3Store } from './lib/store.js';
+
+let defaultOptions = { };
+export default new Ocfl(OcflS3Store, defaultOptions);

ocfl-s3/src/lib/error.ts

@@ -0,0 +1,12 @@
+// Note: So we can return a similar error to node:fs
+export class SystemError extends Error {
+  code: string;
+
+  errno: number;
+
+  constructor(message: string, code: string, errno: number) {
+    super(message);
+    this.code = code;
+    this.errno = errno;
+  }
+}