Cache BCD API bigmap keys

.env

@@ -0,0 +1,2 @@
+DATABASE_URL="file:data.db"
+BCD_API="https://api.better-call.dev"

.gitignore

@@ -31,13 +31,10 @@ node_modules
 
 **/.DS_Store
 **/.vscode
-.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
 
 src/config.json
 
 yarn-debug.log*
 yarn-error.log*
+
+data.db

package.json

@@ -9,24 +9,29 @@
   "dependencies": {
     "@fleekhq/fleek-storage-js": "1.0.14",
     "@pinata/sdk": "1.1.13",
+    "@prisma/client": "2.19.0",
     "@types/bcrypt": "3.0.0",
     "@types/cors": "2.8.10",
     "@types/express": "4.17.2",
     "@types/express-fileupload": "1.1.5",
     "@types/node": "12.12.9",
     "@types/sharp": "0.27.1",
+    "@types/sqlite3": "3.1.7",
     "axios": "0.21.1",
     "cids": "1.0.0",
     "cors": "2.8.5",
+    "dotenv": "8.2.0",
     "express": "4.17.1",
     "express-fileupload": "1.2.0",
     "ipfs-http-client": "47.0.1",
     "lru-cache": "5.1.1",
     "sharp": "0.27.1",
-    "ts-node": "8.5.0",
+    "sqlite3": "5.0.2",
+    "ts-node": "9.1.1",
     "typescript": "3.7.2"
   },
   "devDependencies": {
-    "nodemon": "2.0.4"
+    "nodemon": "2.0.4",
+    "prisma": "2.19.0"
   }
 }

prisma/migrations/20210322223452_init/migration.sql

@@ -0,0 +1,11 @@
+-- CreateTable
+CREATE TABLE "BigMapKey" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "bigMapId" INTEGER NOT NULL,
+    "keyString" TEXT NOT NULL,
+    "data" TEXT NOT NULL,
+    "count" INTEGER NOT NULL
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "BigMapKey.bigMapId_keyString_unique" ON "BigMapKey"("bigMapId", "keyString");

prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "sqlite"

prisma/schema.prisma

@@ -0,0 +1,17 @@
+datasource db {
+  provider = "sqlite"
+  url      = env("DATABASE_URL")
+}
+
+generator client {
+  provider = "prisma-client-js"
+}
+
+model BigMapKey {
+  id         Int    @id @default(autoincrement())
+  bigMapId   Int
+  keyString  String
+  data       String
+  count      Int
+  @@unique([bigMapId, keyString])
+}

src/handlers.ts

@@ -1,6 +1,7 @@
 import { Request, Response } from 'express';
-import fs from 'fs';
 import { IpfsProvider } from './providers/ipfs';
+import axios from 'axios';
+import { PrismaClient } from '@prisma/client';
 
 export async function handleIpfsFileUpload(
   ipfsProvider: IpfsProvider,
@@ -36,7 +37,9 @@ export async function handleIpfsImageWithThumbnailUpload(
   }
 
   try {
-    const content = await ipfsProvider.uploadImageWithThumbnail(file.tempFilePath);
+    const content = await ipfsProvider.uploadImageWithThumbnail(
+      file.tempFilePath
+    );
     return res.status(200).json(content);
   } catch (e) {
     console.log(e);
@@ -64,3 +67,97 @@ export async function handleIpfsJSONUpload(
     });
   }
 }
+
+async function cacheBigMapKeyRange(
+  prisma: PrismaClient,
+  network: string,
+  bigMapId: number,
+  offset: number,
+  size: number
+) {
+  // console.log(`Offset: ${offset}, Size: ${size}`);
+  const queryParams = `?offset=${offset}&size=${size}`;
+  const keysResp = await axios.get(
+    `${process.env.BCD_API}/v1/bigmap/${network}/${bigMapId}/keys${queryParams}`
+  );
+  const data = keysResp.data;
+  for (let item of data) {
+    try {
+      await prisma.bigMapKey.create({
+        data: {
+          bigMapId,
+          keyString: item.data.key_string,
+          data: JSON.stringify(item.data),
+          count: item.count
+        }
+      });
+    } catch (e) {
+      console.log(e);
+    }
+  }
+}
+
+async function cacheBigMapKeys(
+  prisma: PrismaClient,
+  network: string,
+  bigMapId: number
+) {
+  const bigMapResp = await axios.get(
+    `${process.env.BCD_API}/v1/bigmap/${network}/${bigMapId}`
+  );
+  const bigMapKeyCount = await prisma.bigMapKey.count({ where: { bigMapId } });
+  const numUncachedKeys = bigMapResp.data.total_keys - bigMapKeyCount;
+
+  if (numUncachedKeys > 0) {
+    for (let i = numUncachedKeys; i > 0; i = i - 10) {
+      const offset = Math.floor((numUncachedKeys - i) / 10) * 10;
+      const rem = i % 10;
+      const size = rem === 0 || i > 10 ? 10 : rem;
+      await cacheBigMapKeyRange(prisma, network, bigMapId, offset, size);
+    }
+  }
+}
+
+type ParsedCachedBigMapQueryParams =
+  | { valid: false; error: string }
+  | { valid: true; network: string; bigMapId: number };
+
+function parseCachedBigMapQueryParams(
+  req: Request
+): ParsedCachedBigMapQueryParams {
+  const bigMapId = parseInt(req.params.id);
+  if (isNaN(bigMapId)) {
+    return { valid: false, error: 'Failed to parse bigmap id' };
+  }
+
+  const validNetworks = ['mainnet', 'edo2net', 'sandbox'];
+  const network = req.params.network;
+  if (!network || !validNetworks.includes(network)) {
+    return { valid: false, error: `Network '${network}' is not supported` };
+  }
+
+  return { valid: true, bigMapId, network };
+}
+
+export async function handleCachedBigMapQuery(
+  prisma: PrismaClient,
+  req: Request,
+  res: Response
+) {
+  const params = parseCachedBigMapQueryParams(req);
+  if (!params.valid) {
+    return res.status(500).json({ error: params.error });
+  }
+
+  const { network, bigMapId } = params;
+
+  await cacheBigMapKeys(prisma, network, bigMapId);
+
+  const bigMapKeys = await prisma.bigMapKey.findMany({ where: { bigMapId } });
+  const results = bigMapKeys.map(v => ({
+    data: JSON.parse(v.data),
+    count: v.count
+  }));
+
+  return res.status(200).json(results);
+}

src/index.ts

@@ -4,18 +4,25 @@ import fileUpload from 'express-fileupload';
 import http from 'http';
 import fs from 'fs';
 import cors from 'cors';
+import dotenv from 'dotenv';
 import { getProvider } from './helpers/ipfs';
 import {
+  handleCachedBigMapQuery,
   handleIpfsFileUpload,
   handleIpfsImageWithThumbnailUpload,
   handleIpfsJSONUpload
 } from './handlers';
+import { PrismaClient } from '@prisma/client';
+
+dotenv.config();
 
 if (!fs.existsSync('./tmp')) {
   fs.mkdirSync('./tmp');
 }
 
 async function createHttpServer(app: Express) {
+  const prisma = new PrismaClient();
+
   app.use(cors());
   app.use(bodyParser.urlencoded({ extended: true }));
   app.use(bodyParser.json());
@@ -26,8 +33,8 @@ async function createHttpServer(app: Express) {
     })
   );
 
-  app.get('/', (req, res) => {
-    return res.status(200).json({ status: "OK" });
+  app.get('/', (_req, res) => {
+    return res.status(200).json({ status: 'OK' });
   });
 
   const ipfsProvider = await getProvider();
@@ -44,6 +51,10 @@ async function createHttpServer(app: Express) {
     return handleIpfsJSONUpload(ipfsProvider, req, res);
   });
 
+  app.get('/cached-bigmap/:network/:id', (req, res) => {
+    return handleCachedBigMapQuery(prisma, req, res);
+  });
+
   const httpServer = http.createServer(app);
   return httpServer;
 }
@@ -62,7 +73,7 @@ async function run() {
   const app = express();
   const server = await createHttpServer(app);
   server.listen(port, () => {
-    console.log(`[Server] 🚀 Server ready on port ${port}`)
+    console.log(`[Server] 🚀 Server ready on port ${port}`);
   });
 }