Customer -> Identity Data Conversion

[zarathustra323]

Customer to Identity Conversion

• Wipe all cache: rm -rf var/cache/dev/* (if on local)
• Clear cache: bin/console cache:clear (if on local)
• Run JavaScript migration via the mongo shell (see below for current file)
  • Make sure to change the settings for local/prod and for the current app
• Apply the schema: APP=[account]:[app] bin/console as3:modlr:schema:create
• Run question integrations: APP=[account]:[app] bin/console app:integration:question-pull

[zarathustra323]

As a single JavaScript file:

// Settings
const isLocal        = true;
const productionIp   = '10.0.2.50';
const appDbKey       = 'cygnus-vspc';
const service        = {
    name      : 'Cygnus Vehicle Repair Group',
    appId     : '3491BAA7-2B88-485A-93FA-7A5866A7AF5D',
    brandKey  : 'vsp',
    clientKey : 'client_cyg',
    inputId   : '1017F3568790A7K'
};
// End Settings

const externalSource = 'identify:omeda:' + service.brandKey;
const dbName       = 'radix-' + appDbKey;
const dbCopyName   = dbName + '-prod';

function dbExists(name) {
    var admin  = db.getSiblingDB('admin');
    var exists = false;
    var found = admin.runCommand('listDatabases').databases.forEach(function(entry){
        if (entry.name == name) {
            exists = true;
        }
    });
    return exists;
}

if (true === isLocal) {
    if (false === dbExists(dbCopyName)) {
        // Copy from production to local.
        db.copyDatabase(dbName, dbCopyName, productionIp);
    }

    if (false === dbExists(dbName)) {
        db.copyDatabase(dbCopyName, dbName);
    }
}

db = db.getSiblingDB(dbName);
db.getCollectionNames();

// Remove all current indexes
db.getCollectionNames().forEach(function(name) {
   db.runCommand({ dropIndexes: name, index: '*' });
});

// Rename `customer` related collections
db.getCollection("customer").renameCollection("identity");
db.getCollection("customer-email").renameCollection("identity-account-email");
db.getCollection("customer-answer").renameCollection("identity-answer");

// Handle global `identity` update ops.
db.getCollection("identity").update({  }, {
    $unset : { "account" : 1 }
}, { "multi" : true });

// Update account `_type` on `identity` collection
db.getCollection("identity").update({ "_type" : "customer-account" }, {
    $set : { "_type" : "identity-account" }
}, { "multi" : true });

var criteria = {
    "_type"                  : "customer-identity",
    "legacy.external"        : true,
    "externalIds.identifier" : { $exists : true },
};
var updated = 0;

print(db.getCollection('identity').count(criteria));

db.getCollection('identity').find(criteria).forEach(function(doc) {
    var id;
    for (var i = 0; i < doc.externalIds.length; i++) {
        var obj = doc.externalIds[i];
        if (!obj.extra) {
            id = obj.identifier;
        }
    };
    if (id) {
        updated++;
        db.getCollection('identity').update({ "_id" : doc._id }, {
            $set : {
                "_type"      : "identity-external",
                "source"     : externalSource,
                "identifier" : id
            }
        });
    }
});

print(updated);

db.getCollection('identity').update({"_type" : "customer-identity" }, {
    $set : { "_type" : "identity-internal" }
}, { "multi" : true });

// Rename `identity-answer._type` to the new type key
db.getCollection("identity-answer").update({ "_type" : "customer-answer-boolean" }, {
    $set : { "_type" : "identity-answer-boolean" }
}, { "multi" : true });
db.getCollection("identity-answer").update({ "_type" : "customer-answer-choice" }, {
    $set : { "_type" : "identity-answer-choice" }
}, { "multi" : true });
db.getCollection("identity-answer").update({ "_type" : "customer-answer-choices" }, {
    $set : { "_type" : "identity-answer-choices" }
}, { "multi" : true });
db.getCollection("identity-answer").update({ "_type" : "customer-answer-date" }, {
    $set : { "_type" : "identity-answer-date" }
}, { "multi" : true });
db.getCollection("identity-answer").update({ "_type" : "customer-answer-float" }, {
    $set : { "_type" : "identity-answer-float" }
}, { "multi" : true });
db.getCollection("identity-answer").update({ "_type" : "customer-answer-integer" }, {
    $set : { "_type" : "identity-answer-integer" }
}, { "multi" : true });
db.getCollection("identity-answer").update({ "_type" : "customer-answer-string" }, {
    $set : { "_type" : "identity-answer-string" }
}, { "multi" : true });

// Rename customer reference on `identity-answer` collection
db.getCollection("identity-answer").update({  }, {
    $rename : { "customer" : "identity" }
}, { "multi" : true });

// Update `identity-answer` identity account `_type` references
db.getCollection("identity-answer").update({ "identity._type" : "customer-account" }, {
    $set : { "identity._type" : "identity-account" }
}, { "multi" : true });

db.getCollection('identity').aggregate([
    { $match : { "_type" : { $in : [ "identity-internal", "identity-external" ] } } },
    { $group : { "_id" : "$_type", "ids" :  { $push : "$_id" } } } 
]).forEach(function(doc) {
    var type = doc._id;
    db.getCollection('identity-answer').update({ "identity._id" : { $in : doc.ids } }, {
        $set : { "identity._type" : type }
    }, { "multi" : true }); 
});

// Move `primaryEmail` to the `emails` object for internal and external identities
db.getCollection('identity').find({
    "_type"        : { $in : [ 'identity-internal', 'identity-external' ] },
    "primaryEmail" : { $exists: true }
}).forEach(function(doc) {
    var email = {
        identifier : ObjectId().valueOf(),
        isPrimary  : true,
        value      : doc.primaryEmail
    }
    db.getCollection('identity').update({ "_id" : doc._id }, {
        $push  : { "emails": email },
        $unset : { "primaryEmail" : 1 }
    }, { "multi" : true })
});

// Move address data from `customer-address` to the `identity.addresses` field, then drop
db.getCollection('customer-address').find().forEach(function(doc) {
    var id      = doc.customer._id;
    var fields  = [ 'street', 'extra', 'city', 'regionCode', 'postalCode', 'countryCode' ];
    var address = {
        identifier: ObjectId().valueOf()
    };
    for (var i = 0; i < fields.length; i++) {
        var key = fields[i];
        if (doc.hasOwnProperty(key) && doc[key]) {
            address[key] = doc[key];
        }
    };
    db.getCollection('identity').update({ "_id" : id }, {
        $push : { 'addresses' : address }
    });
});

db.getCollection('customer-address').drop();

// Update `input-submission` data
// Rename customer fields to identity
db.getCollection('input-submission').update({}, {
    $rename : { "customer" : "identity", "payload.customer" : "payload.identity" }
}, { "multi" : true });

// Change source keys
var sourceKeys = {
    'customer-account'                         : 'identity-account',
    'customer-email.verify-submit'             : 'identity-account-email.verify-submit',
    'customer-account.reset-password-generate' : 'identity-account.reset-password-generate',
    'customer-account.reset-password'          : 'identity-account.reset-password',
    'customer-email.verify-generate'           : 'identity-account-email.verify-generate'
}
for (var oldKey in sourceKeys) {
    if (!sourceKeys.hasOwnProperty(oldKey)) {
        continue;
    }
    var newKey = sourceKeys[oldKey];
    db.getCollection('input-submission').update({ "sourceKey" : oldKey }, {
        $set : { "sourceKey" : newKey }
    }, { "multi" : true });
}

// Change customer-account type to identity-account type
db.getCollection('input-submission').update({ "identity._type" : "customer-account" }, {
    $set : { "identity._type" : "identity-account" }
}, { "multi" : true });

// Change customer-identity types to the corresponding internal/external type
db.getCollection('identity').aggregate([
    { $match : { "_type" : { $in : [ "identity-internal", "identity-external" ] } } },
    { $group : { "_id" : "$_type", "ids" :  { $push : "$_id" } } } 
]).forEach(function(doc) {
    var type = doc._id;
    db.getCollection('input-submission').update({ "identity._id" : { $in : doc.ids } }, {
        $set : { "identity._type" : type }
    }, { "multi" : true }); 
});

db.getCollection('input-submission').distinct("identity._type");

// Create `integration-service`
db.getCollection('integration-service').insert({
    "_id"         : ObjectId("580fd58b4e993cf47b7362fd"),
    "_type"       : "integration-service-omeda",
    "name"        : service.name,
    "appId"       : service.appId,
    "brandKey"    : service.brandKey,
    "clientKey"   : service.clientKey,
    "inputId"     : service.inputId,
    "createdDate" : ISODate("2016-10-26T21:48:13.549Z"),
    "touchedDate" : ISODate("2016-10-26T21:48:13.549Z"),
    "updatedDate" : ISODate("2016-10-26T21:48:13.549Z")
});

// Drop `integration-client` collection
db.getCollection('integration-client').drop();

// Create `integration-identify` model
db.getCollection('integration').insert({
    "_id"         : ObjectId("580fd86c4e993cf47b7362fe"),
    "_type"       : "integration-identify",
    "pullKey"     : "omeda",
    "name"        : "Omeda Identity Detection",
    "enabled"     : true,
    "service"     : {
        "_id"   : ObjectId("580fd58b4e993cf47b7362fd"),
        "_type" : "integration-service-omeda"
    },
    "createdDate" : ISODate("2016-10-26T21:48:13.549Z"),
    "touchedDate" : ISODate("2016-10-26T21:48:13.549Z"),
    "updatedDate" : ISODate("2016-10-26T21:48:13.549Z")
});

// Convert `question-integration-pull` models to `integration-question-pull` models
db.getCollection('question-integration').update({}, { 
    $unset : { "client": 1 }, 
    $set : { 
        "service" : { "_id": ObjectId("580fd58b4e993cf47b7362fd"), "_type": "integration-service-omeda" },
        "_type"   : "integration-question-pull",
        "boundTo" : "identity",
        "enabled" : true
    }
}, { "multi" : true });

// Move `question-integration-pull` models to` integration` collection
db.getCollection('question-integration').find().forEach(function(doc) {
    db.getCollection('integration').insert(doc);
});

// Drop `question-integration` collection
db.getCollection('question-integration').drop();

// Update `question` boundTo value

db.getCollection('question').update({ "boundTo" : "customer" }, {
    $set : { "boundTo" : "identity" }
}, { "multi" : true });

// Unset `question.builtIn` value
db.getCollection('question').update({}, {
    $unset : { "builtIn": 1 }
}, { "multi" : true });

// Update `question-choice` models to use new integration embed
db.getCollection('question-choice').find({ "integration.clientKey" : { $exists: true } }).forEach(function(doc) {
    db.getCollection('question-choice').update({ "_id" : doc._id }, {
        $set : { 
            "integration.pull.service"    : doc.integration.clientKey,
            "integration.pull.identifier" : doc.integration.identifier,
        },
        $unset : {
            "integration.clientKey"  : 1,
            "integration.identifier" : 1
        }
    }, { "multi" : true });
});

// Change pull meta to use the integration id
db.getCollection('integration').find({ "_type" : "integration-question-pull" }).forEach(function(doc) {
    var question = db.getCollection('question').find({ "pull" : doc._id }).toArray()[0];
    db.getCollection('question-choice').find({ "question" : question._id }).forEach(function(choice) {
        db.getCollection('question-choice').update({ "_id" : choice._id }, {
            $set   : { "integration.pull.integrationId" : doc._id.valueOf() },
            $unset : { "integration.pull.service" : 1 }
        });
    });
});

// Remove `identity-external` duplicates
db.getCollection('identity').aggregate([
    { $match : { "_type" : "identity-external" } },
    { $group : { "_id" : "$identifier", "ids": { $push: "$_id" } } },
    { $match : { "ids.1" : { $exists: true } } }
]).forEach(function(doc) {
    var keep;
    var discard = [];
    db.getCollection('identity').find({ "_id" : { $in: doc.ids } }).forEach(function(identity) {
        if (!keep) {
            keep = identity;
        } else {
            if (identity.createdDate > keep.createdDate) {
                discard.push(keep._id);
                keep = identity;
            } else {
                discard.push(identity._id);
            }
        }
    });
    db.getCollection('identity').remove({ "_id" : { $in: discard } });
});

// Add the Omeda push integration
db.getCollection('integration').insert({
    "_id"         : ObjectId("58177e534e993cf47b7362ff"),
    "_type"       : "integration-account-push",
    "pullKey"     : "omeda",
    "name"        : "Omeda Account Push",
    "enabled"     : true,
    "service"     : {
        "_id"   : ObjectId("580fd58b4e993cf47b7362fd"),
        "_type" : "integration-service-omeda"
    },
    "createdDate" : ISODate("2016-10-31T17:24:13.549Z"),
    "touchedDate" : ISODate("2016-10-31T17:24:13.549Z"),
    "updatedDate" : ISODate("2016-10-31T17:24:13.549Z")
});

// Update `identity-account.externalIds` to push meta
db.getCollection('identity').find({
    "_type"                    : "identity-account",
    "externalIds.0.identifier" : { $exists : true }
}).forEach(function(doc) {
    db.getCollection('identity').update({ "_id" : doc._id }, {
        $push  : { "integration.push" : {
            "integrationId" : "58177e534e993cf47b7362ff",
            "identifier"    : doc.externalIds[0].identifier 
        } },
        $unset : { "externalIds" : 1 }
    });
});
;