From 55816e54f1e5b55bddd08f5ba22e75a88b8a494c Mon Sep 17 00:00:00 2001 From: djvs Date: Thu, 20 Sep 2018 12:18:37 -0400 Subject: [PATCH 1/5] update attestion location, create attestation data model --- .../20180920161631-create-attestation-data.js | 39 +++++++++++++++++++ .../attestations/validateAttestParams.ts | 2 +- .../attestations/validateJobDetails.test.ts | 2 +- app/shared/attestations/validateJobDetails.ts | 2 +- .../whisperAttesterActions.test.ts | 2 +- .../attestations/whisperMessageTypes.ts | 2 +- .../attestations/whisperPersistDataHandler.ts | 2 +- app/shared/ethereum/signingLogic.ts | 2 +- .../models/{Attestations => }/Attestation.ts | 0 .../models/{Attestations => }/Negotiation.ts | 0 .../{Attestations => }/NegotiationMsg.ts | 0 .../{Attestations => }/WhisperFilters.ts | 0 app/shared/models/attestationdata.js | 14 +++++++ .../{Attestations => }/generateNonce.ts | 0 app/shared/models/index.ts | 8 ++-- app/worker/jobs/submit-attestation.ts | 2 +- 16 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 app/migrations/20180920161631-create-attestation-data.js rename app/shared/models/{Attestations => }/Attestation.ts (100%) rename app/shared/models/{Attestations => }/Negotiation.ts (100%) rename app/shared/models/{Attestations => }/NegotiationMsg.ts (100%) rename app/shared/models/{Attestations => }/WhisperFilters.ts (100%) create mode 100644 app/shared/models/attestationdata.js rename app/shared/models/{Attestations => }/generateNonce.ts (100%) diff --git a/app/migrations/20180920161631-create-attestation-data.js b/app/migrations/20180920161631-create-attestation-data.js new file mode 100644 index 0000000..c782ec2 --- /dev/null +++ b/app/migrations/20180920161631-create-attestation-data.js @@ -0,0 +1,39 @@ +'use strict'; +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.createTable('AttestationData', { + id: { + allowNull: false, + autoIncrement: true, + primaryKey: true, + type: Sequelize.INTEGER + }, + datatype: { + type: Sequelize.STRING + }, + data_text: { + type: Sequelize.TEXT + }, + data_blob: { + type: Sequelize.BLOB + }, + challenge: { + type: Sequelize.TEXT + }, + attestation_id: { + type: Sequelize.UUID + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE + } + }); + }, + down: (queryInterface, Sequelize) => { + return queryInterface.dropTable('AttestationData'); + } +}; \ No newline at end of file diff --git a/app/shared/attestations/validateAttestParams.ts b/app/shared/attestations/validateAttestParams.ts index 895a3e7..61277eb 100644 --- a/app/shared/attestations/validateAttestParams.ts +++ b/app/shared/attestations/validateAttestParams.ts @@ -9,7 +9,7 @@ import { getFormattedTypedDataReleaseTokensLegacy, } from '@shared/ethereum/signingLogic' import {AttestationTypeID} from 'attestations-lib' -import {IAttestationDataJSONB} from '@shared/models/Attestations/Attestation' +import {IAttestationDataJSONB} from '@shared/models/Attestation' import BigNumber from 'bignumber.js' import {requiredField} from '@shared/requiredField' import {serverLogger} from '@shared/logger' diff --git a/app/shared/attestations/validateJobDetails.test.ts b/app/shared/attestations/validateJobDetails.test.ts index 92acab4..aa26d07 100644 --- a/app/shared/attestations/validateJobDetails.test.ts +++ b/app/shared/attestations/validateJobDetails.test.ts @@ -9,7 +9,7 @@ import { hashCompleteAttestationData, signAttestationRequest, } from '@shared/ethereum/signingLogic' -import {IAttestationData} from '@shared/models/Attestations/Attestation' +import {IAttestationData} from '@shared/models/Attestation' beforeEach(() => { jest.clearAllMocks() diff --git a/app/shared/attestations/validateJobDetails.ts b/app/shared/attestations/validateJobDetails.ts index 1fbd5b9..11fa3b2 100644 --- a/app/shared/attestations/validateJobDetails.ts +++ b/app/shared/attestations/validateJobDetails.ts @@ -16,7 +16,7 @@ import { import { IAttestationData, IAttestationDataJSONB, -} from '@shared/models/Attestations/Attestation' +} from '@shared/models/Attestation' import {requiredField} from '@shared/requiredField' import {every} from 'lodash' diff --git a/app/shared/attestations/whisperAttesterActions.test.ts b/app/shared/attestations/whisperAttesterActions.test.ts index 40971cb..f8ac13b 100644 --- a/app/shared/attestations/whisperAttesterActions.test.ts +++ b/app/shared/attestations/whisperAttesterActions.test.ts @@ -20,7 +20,7 @@ import { hashCompleteAttestationData, signAttestationRequest, } from '@shared/ethereum/signingLogic' -import {IAttestationData} from '@shared/models/Attestations/Attestation' +import {IAttestationData} from '@shared/models/Attestation' import { attesterWallet, requesterWallet, diff --git a/app/shared/attestations/whisperMessageTypes.ts b/app/shared/attestations/whisperMessageTypes.ts index e41cd10..08f0b54 100644 --- a/app/shared/attestations/whisperMessageTypes.ts +++ b/app/shared/attestations/whisperMessageTypes.ts @@ -1,4 +1,4 @@ -import {IAttestationDataJSONB} from '@shared/models/Attestations/Attestation' +import {IAttestationDataJSONB} from '@shared/models/Attestation' export enum MessageTypes { solicitation = 'solicitation', diff --git a/app/shared/attestations/whisperPersistDataHandler.ts b/app/shared/attestations/whisperPersistDataHandler.ts index 378b732..978e711 100644 --- a/app/shared/attestations/whisperPersistDataHandler.ts +++ b/app/shared/attestations/whisperPersistDataHandler.ts @@ -1,7 +1,7 @@ import BigNumber from 'bignumber.js' import {Negotiation, NegotiationMsg, Attestation} from '@shared/models' import {toBuffer} from 'ethereumjs-util' -import {IAttestationDataJSONB} from '@shared/models/Attestations/Attestation' +import {IAttestationDataJSONB} from '@shared/models/Attestation' import {serverLogger} from '@shared/logger' export enum PersistDataTypes { diff --git a/app/shared/ethereum/signingLogic.ts b/app/shared/ethereum/signingLogic.ts index 707541e..9c38093 100644 --- a/app/shared/ethereum/signingLogic.ts +++ b/app/shared/ethereum/signingLogic.ts @@ -1,7 +1,7 @@ const ethSigUtil = require('eth-sig-util') const {soliditySha3} = require('web3-utils') import {AttestationTypeID} from 'attestations-lib' -import {IAttestationData} from '@shared/models/Attestations/Attestation' +import {IAttestationData} from '@shared/models/Attestation' const uuid = require('uuidv4') import {bufferToHex} from 'ethereumjs-util' diff --git a/app/shared/models/Attestations/Attestation.ts b/app/shared/models/Attestation.ts similarity index 100% rename from app/shared/models/Attestations/Attestation.ts rename to app/shared/models/Attestation.ts diff --git a/app/shared/models/Attestations/Negotiation.ts b/app/shared/models/Negotiation.ts similarity index 100% rename from app/shared/models/Attestations/Negotiation.ts rename to app/shared/models/Negotiation.ts diff --git a/app/shared/models/Attestations/NegotiationMsg.ts b/app/shared/models/NegotiationMsg.ts similarity index 100% rename from app/shared/models/Attestations/NegotiationMsg.ts rename to app/shared/models/NegotiationMsg.ts diff --git a/app/shared/models/Attestations/WhisperFilters.ts b/app/shared/models/WhisperFilters.ts similarity index 100% rename from app/shared/models/Attestations/WhisperFilters.ts rename to app/shared/models/WhisperFilters.ts diff --git a/app/shared/models/attestationdata.js b/app/shared/models/attestationdata.js new file mode 100644 index 0000000..e95fe71 --- /dev/null +++ b/app/shared/models/attestationdata.js @@ -0,0 +1,14 @@ +'use strict'; +module.exports = (sequelize, DataTypes) => { + var AttestationData = sequelize.define('AttestationData', { + datatype: DataTypes.STRING, + data_text: DataTypes.TEXT, + data_blob: DataTypes.BLOB, + challenge: DataTypes.TEXT, + attestation_id: DataTypes.UUID + }, {}); + AttestationData.associate = function(models) { + // associations can be defined here + }; + return AttestationData; +}; \ No newline at end of file diff --git a/app/shared/models/Attestations/generateNonce.ts b/app/shared/models/generateNonce.ts similarity index 100% rename from app/shared/models/Attestations/generateNonce.ts rename to app/shared/models/generateNonce.ts diff --git a/app/shared/models/index.ts b/app/shared/models/index.ts index e6e1376..9611fd1 100644 --- a/app/shared/models/index.ts +++ b/app/shared/models/index.ts @@ -1,10 +1,10 @@ import {Sequelize} from 'sequelize-typescript' import {env} from '@shared/environment' import * as config from '../../config/database' -import Negotiation from '@shared/models/Attestations/Negotiation' -import NegotiationMsg from '@shared/models/Attestations/NegotiationMsg' -import WhisperFilters from '@shared/models/Attestations/WhisperFilters' -import Attestation from '@shared/models/Attestations/Attestation' +import Negotiation from '@shared/models/Negotiation' +import NegotiationMsg from '@shared/models/NegotiationMsg' +import WhisperFilters from '@shared/models/WhisperFilters' +import Attestation from '@shared/models/Attestation' import GasPrice from '@shared/models/GasPrice' const environmentConfig = config[env.nodeEnv] diff --git a/app/worker/jobs/submit-attestation.ts b/app/worker/jobs/submit-attestation.ts index db7b7b5..06fbdac 100644 --- a/app/worker/jobs/submit-attestation.ts +++ b/app/worker/jobs/submit-attestation.ts @@ -2,7 +2,7 @@ import {attesterWallet} from '@shared/attestations/attestationWallets' import * as newrelic from 'newrelic' import {Attestation} from '@shared/models' import {sendAttestTx} from '@shared/attestations/sendAttest' -import {IAttestationResult} from '@shared/models/Attestations/Attestation' +import {IAttestationResult} from '@shared/models/Attestation' import {notifyAttestationCompleted} from '@shared/webhookHandler' import {serverLogger} from '@shared/logger' From 47440584a932f18237dd33c961b7a1649b2a944a Mon Sep 17 00:00:00 2001 From: djvs Date: Thu, 20 Sep 2018 13:30:53 -0400 Subject: [PATCH 2/5] typescript model & integration --- .../20180920161631-create-attestation-data.js | 61 ++++++++--------- app/shared/models/AttestationData.ts | 66 +++++++++++++++++++ app/shared/models/attestationdata.js | 14 ---- 3 files changed, 92 insertions(+), 49 deletions(-) create mode 100644 app/shared/models/AttestationData.ts delete mode 100644 app/shared/models/attestationdata.js diff --git a/app/migrations/20180920161631-create-attestation-data.js b/app/migrations/20180920161631-create-attestation-data.js index c782ec2..4b99414 100644 --- a/app/migrations/20180920161631-create-attestation-data.js +++ b/app/migrations/20180920161631-create-attestation-data.js @@ -1,39 +1,30 @@ -'use strict'; +'use strict' module.exports = { up: (queryInterface, Sequelize) => { - return queryInterface.createTable('AttestationData', { - id: { - allowNull: false, - autoIncrement: true, - primaryKey: true, - type: Sequelize.INTEGER - }, - datatype: { - type: Sequelize.STRING - }, - data_text: { - type: Sequelize.TEXT - }, - data_blob: { - type: Sequelize.BLOB - }, - challenge: { - type: Sequelize.TEXT - }, - attestation_id: { - type: Sequelize.UUID - }, - createdAt: { - allowNull: false, - type: Sequelize.DATE - }, - updatedAt: { - allowNull: false, - type: Sequelize.DATE - } - }); + return queryInterface.sequelize.query(` + CREATE TYPE "attestation_data_datatype" + AS ENUM('text','blob'); + + CREATE TABLE "attestationData" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "created" timestamp without time zone DEFAULT now() NOT NULL, + "updated" timestamp without time zone DEFAULT now() NOT NULL, + + "attestationId" uuid, + "messageType" enum_whisper_msg_types, + + "datatype" attestation_data_datatype, + "dtext" text, + "dblob" blob, + + "challenge" varchar(256) + ); + `) }, down: (queryInterface, Sequelize) => { - return queryInterface.dropTable('AttestationData'); - } -}; \ No newline at end of file + return queryInterface.sequelize.query(` + drop table AttestationData; + drop type attestation_data_datatype; + `) + }, +} diff --git a/app/shared/models/AttestationData.ts b/app/shared/models/AttestationData.ts new file mode 100644 index 0000000..6c9f5d1 --- /dev/null +++ b/app/shared/models/AttestationData.ts @@ -0,0 +1,66 @@ +import * as Sequelize from 'sequelize-typescript' +import {Attestation} from '@shared/models' +import {WhisperMsgDataType} from '@shared/models/NegotiationMsg' + +@Sequelize.Table({tableName: 'attestation_data'}) +export default class AttestationData extends Sequelize.Model { + @Sequelize.Column({ + type: Sequelize.DataType.UUID, + allowNull: false, + autoIncrement: false, + primaryKey: true, + }) + id: string + + @Sequelize.CreatedAt created: Date + + @Sequelize.UpdatedAt updated: Date + + @Sequelize.Column({ + type: Sequelize.DataType.UUID, + allowNull: false, + autoIncrement: false, + primaryKey: true, + }) + attestationId: string + + @Sequelize.BelongsTo(() => Attestation, { + foreignKey: 'attestationId', + }) + Attestation?: Attestation + + // Column to specify data storage type + @Sequelize.Column({ + allowNull: false, + type: 'enum_whisper_msg_types', + }) + messageType: WhisperMsgDataType + + // Column to specify data storage type + @Sequelize.Column({ + allowNull: false, + type: Sequelize.DataType.STRING, + }) + datatype: 'text' | 'blob' + + // Column for data stored as text + @Sequelize.Column({ + allowNull: false, + type: Sequelize.DataType.TEXT, + }) + dtext: string + + // Column for data stored as blob + @Sequelize.Column({ + allowNull: false, + type: Sequelize.DataType.BLOB, + }) + dblob: Buffer + + // Cryptographic challenge (SHA256'ed password) + @Sequelize.Column({ + allowNull: false, + type: Sequelize.DataType.STRING, + }) + challenge: string +} diff --git a/app/shared/models/attestationdata.js b/app/shared/models/attestationdata.js deleted file mode 100644 index e95fe71..0000000 --- a/app/shared/models/attestationdata.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; -module.exports = (sequelize, DataTypes) => { - var AttestationData = sequelize.define('AttestationData', { - datatype: DataTypes.STRING, - data_text: DataTypes.TEXT, - data_blob: DataTypes.BLOB, - challenge: DataTypes.TEXT, - attestation_id: DataTypes.UUID - }, {}); - AttestationData.associate = function(models) { - // associations can be defined here - }; - return AttestationData; -}; \ No newline at end of file From 1453af17546d0fd2071f0b0eb696ee3cf83afacf Mon Sep 17 00:00:00 2001 From: djvs Date: Thu, 20 Sep 2018 13:54:48 -0400 Subject: [PATCH 3/5] attestation data api --- app/api/controllers/attestation_data.ts | 50 +++++++++++++++++++++++++ app/api/index.ts | 9 ++++- app/shared/models/AttestationData.ts | 6 +++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 app/api/controllers/attestation_data.ts diff --git a/app/api/controllers/attestation_data.ts b/app/api/controllers/attestation_data.ts new file mode 100644 index 0000000..7554e0b --- /dev/null +++ b/app/api/controllers/attestation_data.ts @@ -0,0 +1,50 @@ +import * as m from '@shared/models' +import {serverLogger} from '@shared/logger' +import {bufferToHex, sha256} from 'ethereumjs-util' +import * as dc from 'deepcopy' + +// perform attestation on an existing request +export const show = async (req: any, res: any) => { + const ad = await m.AttestationData.findById(req.params.attestation_data_id) + + if (!ad) { + return res.status(404).json({success: false, message: 'Not found'}) + } + + const adRaw = ad.get({raw: true}) + + if (!ad.testChallenge(req.body.passphrase)) { + res + .status(401) + .json({success: false, message: 'Invalid credentials for attestation data'}) + } + + res.json({ + success: true, + attestation_data: { + id: adRaw.id, + created: adRaw.created, + updated: adRaw.updated, + messageType: adRaw.messageType, + data: adRaw.datatype === 'text' ? adRaw.dtext : adRaw.dblob, + datatype: adRaw.datatype, + messageType: adRaw.messageType, + }, + }) +} + +export const destroy = async (req: any, res: any) => { + const ad = await m.AttestationData.findById(req.params.attestation_data_id) + + if (!ad) { + res.status(404).json({success: false, message: 'Not found'}) + } + + if (!ad.testChallenge(req.body.passphrase)) { + res + .status(401) + .json({success: false, message: 'Invalid credentials for attestation data'}) + } + + await ad.destroy() +} diff --git a/app/api/index.ts b/app/api/index.ts index 0d4853b..f0df34c 100644 --- a/app/api/index.ts +++ b/app/api/index.ts @@ -1,11 +1,13 @@ import * as express from 'express' // import * as path from 'path' import {env} from '@shared/environment' -import * as attCtrl from '@api/controllers/attestation' -import * as reqCtrl from '@api/controllers/request' import {sha256} from 'ethereumjs-util' import * as bodyParser from 'body-parser' +import * as attCtrl from '@api/controllers/attestation' +import * as reqCtrl from '@api/controllers/request' +import * as atDatCtrl from '@api/controllers/attestation_data' + const app = express() interface IRequestWithRawBody extends express.Request { @@ -49,6 +51,9 @@ app.post('/api/requests/send', reqCtrl.sendjob) app.get('/api/attestations', attCtrl.show) app.post('/api/attestations', attCtrl.perform) +app.get('/api/attestation_data/:attestation_data_id', atDatCtrl.show) +app.delete('/api/attestation_data/:attestation_data_id', atDatCtrl.destroy) + app.listen(13000, () => console.log('App listening on port 13000')) export default app diff --git a/app/shared/models/AttestationData.ts b/app/shared/models/AttestationData.ts index 6c9f5d1..98bfa95 100644 --- a/app/shared/models/AttestationData.ts +++ b/app/shared/models/AttestationData.ts @@ -1,6 +1,7 @@ import * as Sequelize from 'sequelize-typescript' import {Attestation} from '@shared/models' import {WhisperMsgDataType} from '@shared/models/NegotiationMsg' +import {bufferToHex, sha256} from 'ethereumjs-util' @Sequelize.Table({tableName: 'attestation_data'}) export default class AttestationData extends Sequelize.Model { @@ -63,4 +64,9 @@ export default class AttestationData extends Sequelize.Model { type: Sequelize.DataType.STRING, }) challenge: string + + testChallenge = (passphrase: str) => { + let passphraseHash = bufferToHex(sha256(passphrase)) + return passphrase && this.challenge && passphraseHash === this.challenge + } } From 5b91e88edcb2f6c56bd713883f5a737831997bdd Mon Sep 17 00:00:00 2001 From: djvs Date: Mon, 24 Sep 2018 12:06:31 -0400 Subject: [PATCH 4/5] fix --- app/api/controllers/attestation_data.ts | 12 +++---- app/package.json | 1 + .../attestations/validateAttestParams.ts | 4 +-- app/shared/attestations/validateJobDetails.ts | 17 ++++++++-- .../attestations/whisperAttesterActions.ts | 33 ++++++++++++++++++- .../attestations/whisperMessageTypes.ts | 7 ++-- .../attestations/whisperPersistDataHandler.ts | 4 +-- .../attestations/whisperRequesterActions.ts | 28 +++++++++++++++- app/shared/environment.ts | 2 ++ app/shared/models/Attestation.ts | 24 ++++++++++++-- app/shared/models/AttestationData.ts | 6 ++-- app/shared/models/index.ts | 3 ++ app/yarn.lock | 15 ++++++++- 13 files changed, 130 insertions(+), 26 deletions(-) diff --git a/app/api/controllers/attestation_data.ts b/app/api/controllers/attestation_data.ts index 7554e0b..77bd9c8 100644 --- a/app/api/controllers/attestation_data.ts +++ b/app/api/controllers/attestation_data.ts @@ -1,7 +1,4 @@ import * as m from '@shared/models' -import {serverLogger} from '@shared/logger' -import {bufferToHex, sha256} from 'ethereumjs-util' -import * as dc from 'deepcopy' // perform attestation on an existing request export const show = async (req: any, res: any) => { @@ -11,9 +8,9 @@ export const show = async (req: any, res: any) => { return res.status(404).json({success: false, message: 'Not found'}) } - const adRaw = ad.get({raw: true}) + const adRaw = ad.get({plain: true}) - if (!ad.testChallenge(req.body.passphrase)) { + if (!ad.testChallenge(req.query.passphrase)) { res .status(401) .json({success: false, message: 'Invalid credentials for attestation data'}) @@ -25,7 +22,6 @@ export const show = async (req: any, res: any) => { id: adRaw.id, created: adRaw.created, updated: adRaw.updated, - messageType: adRaw.messageType, data: adRaw.datatype === 'text' ? adRaw.dtext : adRaw.dblob, datatype: adRaw.datatype, messageType: adRaw.messageType, @@ -37,10 +33,10 @@ export const destroy = async (req: any, res: any) => { const ad = await m.AttestationData.findById(req.params.attestation_data_id) if (!ad) { - res.status(404).json({success: false, message: 'Not found'}) + return res.status(404).json({success: false, message: 'Not found'}) } - if (!ad.testChallenge(req.body.passphrase)) { + if (!ad.testChallenge(req.query.passphrase)) { res .status(401) .json({success: false, message: 'Invalid credentials for attestation data'}) diff --git a/app/package.json b/app/package.json index a7b3559..4c2c023 100644 --- a/app/package.json +++ b/app/package.json @@ -34,6 +34,7 @@ "@types/winston": "^2.3.7", "async": "^2.6.0", "attestations-lib": "https://github.com/hellobloom/attestations-lib/releases/download/v0.2.1/attestations-lib-v0.2.1.tgz", + "axios": "^0.18.0", "babel-preset-stage-3": "^6.24.1", "babel-register": "^6.26.0", "bignumber.js": "^5.0.0", diff --git a/app/shared/attestations/validateAttestParams.ts b/app/shared/attestations/validateAttestParams.ts index 61277eb..91c613a 100644 --- a/app/shared/attestations/validateAttestParams.ts +++ b/app/shared/attestations/validateAttestParams.ts @@ -9,7 +9,7 @@ import { getFormattedTypedDataReleaseTokensLegacy, } from '@shared/ethereum/signingLogic' import {AttestationTypeID} from 'attestations-lib' -import {IAttestationDataJSONB} from '@shared/models/Attestation' +import {TAttestationDataJSONB} from '@shared/models/Attestation' import BigNumber from 'bignumber.js' import {requiredField} from '@shared/requiredField' import {serverLogger} from '@shared/logger' @@ -34,7 +34,7 @@ export interface IUnvalidatedAttestParams { reward: BigNumber paymentNonce: string requesterSig: string - data: IAttestationDataJSONB + data: TAttestationDataJSONB types: AttestationTypeID[] requestNonce: string subjectSig: string diff --git a/app/shared/attestations/validateJobDetails.ts b/app/shared/attestations/validateJobDetails.ts index 11fa3b2..8d300f9 100644 --- a/app/shared/attestations/validateJobDetails.ts +++ b/app/shared/attestations/validateJobDetails.ts @@ -15,7 +15,8 @@ import { } from '@shared/ethereum/signingLogic' import { IAttestationData, - IAttestationDataJSONB, + TMaybeAttestationDataJSONB, + TAttestationDataJSONB, } from '@shared/models/Attestation' import {requiredField} from '@shared/requiredField' import {every} from 'lodash' @@ -34,8 +35,18 @@ interface ISuccess { export type TValidateJobDetailsOutput = IInvalidParamError | ISuccess +export interface IMaybeJobDetails { + data: TMaybeAttestationDataJSONB + requestNonce: string + types: number[] + subject: string + subjectSig: string + attester: string + requester: string +} + export interface IJobDetails { - data: IAttestationDataJSONB + data: TMaybeAttestationDataJSONB requestNonce: string types: number[] subject: string @@ -241,7 +252,7 @@ export const validateSubjectDataComponent = ( } export const validateSubjectData = ( - input: IAttestationDataJSONB, + input: TAttestationDataJSONB, type: AttestationTypeID[] ): boolean => { if (!input || input.data.length !== type.length) return false diff --git a/app/shared/attestations/whisperAttesterActions.ts b/app/shared/attestations/whisperAttesterActions.ts index 73fe8b8..2912fab 100644 --- a/app/shared/attestations/whisperAttesterActions.ts +++ b/app/shared/attestations/whisperAttesterActions.ts @@ -38,7 +38,9 @@ import { ExternalActionTypes, } from '@shared/attestations/whisperExternalActionHandler' import {hashedTopicToAttestationType} from '@shared/attestations/AttestationUtils' +import {TAttestationDataJSONB} from '@shared/models/Attestation' import {env} from '@shared/environment' +import axios from 'axios' import * as Web3 from 'web3' export const listenForSolicitations = async ( @@ -206,17 +208,46 @@ const startAttestation = ( return decision } +// Retrieve the subject data if the message instructs it to be retrieved over HTTPS out-of-band, and then return the modified message if so +export const checkForMessageRetrieval = async (message: ISendJobDetails) => { + try { + let info = message.subjectData as any + if (info.dataHostedExternally) { + var dataObj + if (info.retrievalUrl) { + dataObj = (await axios.get(info.retrievalUrl)).data + if (info.deletionUrl) { + await axios.delete(info.deletionUrl) + } + } else { + var resourceUrl = `https://${info.nodeHost}/attestation_data/${ + info.attestationDataId + }?passphrase=${info.passphrase}` + dataObj = (await axios.get(resourceUrl)).data + await axios.delete(resourceUrl) + } + message.subjectData = JSON.parse(dataObj.data) + } + return message + } catch (error) { + console.log(error) + throw new Error('Asynchronous data retrieval failed') + } +} + export const handleJobDetails: TMessageHandler = async ( message: ISendJobDetails, messageTopic: string, attesterWallet: Wallet.Wallet ) => { + message = await checkForMessageRetrieval(message) + try { let decision: IMessageDecision const _isApprovedRequester = await isApprovedRequester(message) const _rewardMatchesBid = await rewardMatchesBid(message) const _validateSubjectData = validateSubjectData( - message.subjectData, + message.subjectData as TAttestationDataJSONB, message.typeIds ) diff --git a/app/shared/attestations/whisperMessageTypes.ts b/app/shared/attestations/whisperMessageTypes.ts index 08f0b54..6373bac 100644 --- a/app/shared/attestations/whisperMessageTypes.ts +++ b/app/shared/attestations/whisperMessageTypes.ts @@ -1,4 +1,7 @@ -import {IAttestationDataJSONB} from '@shared/models/Attestation' +import { + TAttestationDataJSONB, + TUnresolvedAttestationData, +} from '@shared/models/Attestation' export enum MessageTypes { solicitation = 'solicitation', @@ -53,7 +56,7 @@ export interface ISubmitSubjectData extends IBloomWhisperResponse { export interface ISendJobDetails extends IBloomWhisperResponse { messageType: MessageTypes.sendJobDetails reward: string - subjectData: IAttestationDataJSONB + subjectData: TUnresolvedAttestationData | TAttestationDataJSONB subjectRequestNonce: string typeIds: number[] subjectAddress: string diff --git a/app/shared/attestations/whisperPersistDataHandler.ts b/app/shared/attestations/whisperPersistDataHandler.ts index 978e711..7557b98 100644 --- a/app/shared/attestations/whisperPersistDataHandler.ts +++ b/app/shared/attestations/whisperPersistDataHandler.ts @@ -1,7 +1,7 @@ import BigNumber from 'bignumber.js' import {Negotiation, NegotiationMsg, Attestation} from '@shared/models' import {toBuffer} from 'ethereumjs-util' -import {IAttestationDataJSONB} from '@shared/models/Attestation' +import {TMaybeAttestationDataJSONB} from '@shared/models/Attestation' import {serverLogger} from '@shared/logger' export enum PersistDataTypes { @@ -72,7 +72,7 @@ export interface IStoreJobDetails { subject: string attester: string requester: string - subjectData: IAttestationDataJSONB + subjectData: TMaybeAttestationDataJSONB subjectRequestNonce: string typeIds: number[] type: string diff --git a/app/shared/attestations/whisperRequesterActions.ts b/app/shared/attestations/whisperRequesterActions.ts index d5cfe20..d95a100 100644 --- a/app/shared/attestations/whisperRequesterActions.ts +++ b/app/shared/attestations/whisperRequesterActions.ts @@ -2,7 +2,11 @@ import * as newrelic from 'newrelic' const uuid = require('uuidv4') import BigNumber from 'bignumber.js' import * as Wallet from 'ethereumjs-wallet' -import {toBuffer} from 'ethereumjs-util' +import {bufferToHex, sha256, toBuffer} from 'ethereumjs-util' +import {AttestationData} from '@shared/models' +import {generateNonce} from '@shared/models/generateNonce' +import {IDataRetrievalKit} from '@shared/models/Attestation' +import {env} from '@shared/environment' import { IMessageDecision, @@ -240,6 +244,28 @@ export const sendJobDetails = async ( requesterWallet.getPrivateKey() ) + // Store the subject data in DB and send retrieval details if it's too big + let subjectDataString = JSON.stringify(jobDetails.data.data) + if (subjectDataString.length > 10000) { + var passphrase = generateNonce() + .toString(16) + .substr(0, 64) + const attData = await AttestationData.create({ + attestationId: attestation.id, + challenge: bufferToHex(sha256(passphrase)), + messageType: 'storeSendJobDetails', + datatype: 'text', + dtext: subjectDataString, + }) + const dataRetrievalInstructions: IDataRetrievalKit = { + dataHostedExternally: true, + nodeHost: env.hostname, + attestationDataId: attData.id, + passphrase: passphrase, + } + jobDetails.data.data = dataRetrievalInstructions + } + const messageResponse: ISendJobDetails = { messageType: MessageTypes.sendJobDetails, replyTo: 'new', diff --git a/app/shared/environment.ts b/app/shared/environment.ts index e6e030c..42c6c2a 100644 --- a/app/shared/environment.ts +++ b/app/shared/environment.ts @@ -4,6 +4,7 @@ import * as dotenv from 'dotenv' dotenv.config() interface IEnvironmentConfig { + hostname: string apiKey: string dbUrl: string rinkebyAccountRegistryAddress: string @@ -141,6 +142,7 @@ Object.keys(topics).forEach(k => { }) export const env: IEnvironmentConfig = { + hostname: envVar('HOSTNAME'), apiKey: envVar('API_KEY_SHA256'), dbUrl: envVar('BLOOM_WHISPER_PG_URL'), rinkebyAccountRegistryAddress: envVar('RINKEBY_ACCOUNT_REGISTRY_ADDRESS'), diff --git a/app/shared/models/Attestation.ts b/app/shared/models/Attestation.ts index 0e8bc51..7f9288c 100644 --- a/app/shared/models/Attestation.ts +++ b/app/shared/models/Attestation.ts @@ -42,9 +42,26 @@ export interface IAttestationResult { certainty?: number } +export interface IDataRetrievalKit { + dataHostedExternally: boolean + nodeHost: string + attestationDataId: string + passphrase: string +} + +export interface IDataRetrievalOther { + dataHostedExternally: boolean + retrievalUrl: string + deletionUrl: string +} + export type AttestationRole = 'attester' | 'requester' -export type IAttestationDataJSONB = IEmailAttestationJSONB | IPhoneAttestationJSONB +export type TAttestationDataJSONB = IEmailAttestationJSONB | IPhoneAttestationJSONB + +export type TUnresolvedAttestationData = IDataRetrievalKit | IDataRetrievalOther + +export type TMaybeAttestationDataJSONB = TUnresolvedAttestationData | TAttestationDataJSONB export const AttestationStatusDataType = Sequelize.DataType.ENUM( Object.keys(AttestationStatus) @@ -96,7 +113,7 @@ export default class Attestation extends Sequelize.Model { get data() { return this.getDataValue('data') } - set data(value: IAttestationDataJSONB) { + set data(value: TAttestationDataJSONB) { this.setDataValue('data', value) } @@ -166,7 +183,8 @@ export default class Attestation extends Sequelize.Model { ) } - validateJobDetailsView(): IJobDetails { + validateJobDetailsView = (): IJobDetails => { + if (!this.data || !this.data.data) throw new Error('Invalid data') return { // Making sure the data prop only contains what the validate job details cares about data: { diff --git a/app/shared/models/AttestationData.ts b/app/shared/models/AttestationData.ts index 98bfa95..f29a416 100644 --- a/app/shared/models/AttestationData.ts +++ b/app/shared/models/AttestationData.ts @@ -33,9 +33,9 @@ export default class AttestationData extends Sequelize.Model { // Column to specify data storage type @Sequelize.Column({ allowNull: false, - type: 'enum_whisper_msg_types', + type: WhisperMsgDataType, }) - messageType: WhisperMsgDataType + messageType: string // Column to specify data storage type @Sequelize.Column({ @@ -65,7 +65,7 @@ export default class AttestationData extends Sequelize.Model { }) challenge: string - testChallenge = (passphrase: str) => { + testChallenge = (passphrase: string) => { let passphraseHash = bufferToHex(sha256(passphrase)) return passphrase && this.challenge && passphraseHash === this.challenge } diff --git a/app/shared/models/index.ts b/app/shared/models/index.ts index 9611fd1..5d6b5fc 100644 --- a/app/shared/models/index.ts +++ b/app/shared/models/index.ts @@ -5,6 +5,7 @@ import Negotiation from '@shared/models/Negotiation' import NegotiationMsg from '@shared/models/NegotiationMsg' import WhisperFilters from '@shared/models/WhisperFilters' import Attestation from '@shared/models/Attestation' +import AttestationData from '@shared/models/AttestationData' import GasPrice from '@shared/models/GasPrice' const environmentConfig = config[env.nodeEnv] @@ -29,6 +30,7 @@ sequelize.addModels([ NegotiationMsg, WhisperFilters, Attestation, + AttestationData, GasPrice, ]) @@ -41,5 +43,6 @@ export { sequelize, WhisperFilters, Attestation, + AttestationData, GasPrice, } diff --git a/app/yarn.lock b/app/yarn.lock index be327ae..d5ea705 100644 --- a/app/yarn.lock +++ b/app/yarn.lock @@ -847,6 +847,13 @@ aws4@^1.2.1, aws4@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" +axios@^0.18.0: + version "0.18.0" + resolved "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102" + dependencies: + follow-redirects "^1.3.0" + is-buffer "^1.1.5" + babel-code-frame@^6.22.0, babel-code-frame@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" @@ -2497,7 +2504,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@*, debug@3.1.0, debug@^3.0.0, debug@^3.1.0: +debug@*, debug@3.1.0, debug@=3.1.0, debug@^3.0.0, debug@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: @@ -3598,6 +3605,12 @@ follow-redirects@^1.0.0: dependencies: debug "^3.1.0" +follow-redirects@^1.3.0: + version "1.5.8" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.8.tgz#1dbfe13e45ad969f813e86c00e5296f525c885a1" + dependencies: + debug "=3.1.0" + for-each@^0.3.2, for-each@~0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.2.tgz#2c40450b9348e97f281322593ba96704b9abd4d4" From ff51499e837bc705fceb807dd48ba3a778cadb51 Mon Sep 17 00:00:00 2001 From: djvs Date: Mon, 12 Nov 2018 16:14:20 -0500 Subject: [PATCH 5/5] merge --- app/shared/whisper/attesterActions.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/shared/whisper/attesterActions.ts b/app/shared/whisper/attesterActions.ts index 690162d..386e3e2 100644 --- a/app/shared/whisper/attesterActions.ts +++ b/app/shared/whisper/attesterActions.ts @@ -39,7 +39,7 @@ import {TAttestationDataJSONB} from '@shared/models/Attestation' import {env} from '@shared/environment' import axios from 'axios' import * as Web3 from 'web3' -import {AttestationTypeID, HashingLogic} from '@bloomprotocol/attestations-lib' +// import {AttestationTypeID, HashingLogic} from '@bloomprotocol/attestations-lib' export const listenForSolicitations = async ( listeningTopic: string, @@ -170,9 +170,10 @@ const startAttestation = ( subjectData: message.subjectData, subjectRequestNonce: message.subjectRequestNonce, type: hashedTopicToAttestationType[messageTopic], - typeIds: message.subjectData.data.map( + typeIds: message.typeIds, + /* typeIds: message.subjectData.data.map( (a: HashingLogic.IAttestationData) => AttestationTypeID[a.type] - ), + ), */ subjectSignature: message.subjectSignature, paymentSignature: message.paymentSignature, paymentNonce: message.paymentNonce, @@ -249,10 +250,10 @@ export const handleJobDetails: TMsgHandler = async ( const _rewardMatchesBid = await rewardMatchesBid(message) const _validateSubjectData = validateSubjectData( message.subjectData as TAttestationDataJSONB, - // message.typeIds, - message.subjectData.data.map( + message.typeIds + /* message.subjectData.data.map( (a: HashingLogic.IAttestationData) => AttestationTypeID[a.type] - ) + ) */ ) serverLogger.info(`validate output: ${_validateSubjectData}`)