Merrick -> Radix Import Migration

[zarathustra323]

Before running Merrick import, ensure no duplicate Omeda IDs exist:

var site = "www.forconstructionpros.com";

db.getCollection('users_v2').aggregate([
    {
        $match : {
            site: site,
            $or: [
                { pwd: { $exists: false } },
                { pwd: { $in: [ null, "" ] } }
            ],
            origin: "link_tracking",
            $and: [
                { omeda_id: { $exists: true } },
                { omeda_id: { $nin: [ null, "" ] } }
            ]
        }    
    },
    {
        $group: {
            _id: "$omeda_id",
            emails: { $addToSet: {
               value: "$email",
               created: "$created"
            } },
            count: { $sum: 1 }
        }
    },
    {
        $match: {
            count: { $gt: 1 }
        }
    },
    {
        $unwind: "$emails"
    },
    {
        $sort: { "emails.created": 1 }
    },
    {
        $group: {
            _id: "$_id",
            emails: { $addToSet: "$emails.value" }
        }
    },
]).forEach(function(doc) {
    doc.emails.forEach(function(email, index) {
        if (index > 0) {
            // Kill
            db.getCollection('users_v2').update({
                site: site,
                email: email,
            }, {
                $unset: {
                    omeda_id: 1,
                    omeda_encrypted_id: 1
                }
            });
            print('Unset Omeda IDs for ' + site + ': ' + email);
        }
    });
});

Spot check that answers have been imported correctly (should run multiple times or change the sample size)

var cursor = db.getCollection('identity').aggregate([
  {
    $match : { 'legacy.questions': { $exists: true } }
  },
  {
    $sample: { size: 100 }
  },
  {
    $unwind: '$legacy.questions'
  },
  {
    $lookup: {
      from: 'question-choice',
      localField: 'legacy.questions.answer',
      foreignField: 'integration.pull.identifier',
      as: 'pull'
    }
  },
  {
    $match: { pull: { $ne: [] } }
  },
  {
    $unwind: '$pull',
  },
  {
    $group: {
      _id: '$_id',
      answers: { $addToSet: { question: '$pull.question', value: '$pull._id' } }
    }
  },
]);
cursor.forEach(function(identity) {
  var answerCursor = db.getCollection('identity-answer').find({
    'identity._id': identity._id,
    $or: identity.answers
  }, { _id: 1 });

  if (identity.answers.length != answerCursor.count()) {
    print('ERROR! ID: ' + identity._id + ' Legacy: ' + identity.answers.length + ' Radix: ' + answerCursor.count());
  }
});

Ensure Omeda IDs have been set to accounts and internal identities (should return nothing if OK)

db.getCollection('identity').aggregate([
  {
    $match: { _type: { $in: ['identity-account', 'identity-internal'] }, 'legacy.omeda_id': { $exists: true } }
  },
  {
    $unwind: '$integration.push',
  },
  {
    $project: {
      _type: 1,
      ok: { $eq: [ '$legacy.omeda_id', '$integration.push.identifier' ] }
    }
  },
  {
    $match: { ok: false }
  },
]);

Ensure address, phone, and email identifiers have been set (should return nothing if OK)

db.getCollection('identity').find({
  addresses: { $exists: true },
  $or: [
    { 'addresses.identifier': { $exists: false } },
    { 'addresses.identifier': { $in: [ null, '' ] } },
  ]
});


db.getCollection('identity').find({
  phones: { $exists: true },
  $or: [
    { 'phones.identifier': { $exists: false } },
    { 'phones.identifier': { $in: [ null, '' ] } },
  ]
});

db.getCollection('identity').find({
  emails: { $exists: true },
  $or: [
    { 'emails.identifier': { $exists: false } },
    { 'emails.identifier': { $in: [ null, '' ] } },
  ]
});

Ensure source and identifier have been set on external identities

db.getCollection('identity').find({
  _type: 'identity-external',
  $or: [
    { 'identifier': { $exists: false } },
    { 'source': { $exists: false } },
  ]
});

Spot check industry answers (only applies to sites with industry) MUST SET ALTERNATE IDS TO RADIX QUESTIONS!

var cursor = db.getCollection('identity').aggregate([
  {
    $match : { 'legacy.industry': { $exists: true } }
  },
  {
    $sample: { size: 100 }
  },
  {
    $lookup: {
      from: 'question-choice',
      localField: 'legacy.industry',
      foreignField: 'alternateId',
      as: 'choice'
    }
  },
  {
    $match: { choice: { $ne: [] } }
  },
  {
    $unwind: '$choice',
  },
  {
    $group: {
      _id: '$_id',
      answers: { $addToSet: { question: '$choice.question', value: '$choice._id' } }
    }
  },
]);
cursor.forEach(function(identity) {
  var answerCursor = db.getCollection('identity-answer').find({
    'identity._id': identity._id,
    $or: identity.answers
  }, { _id: 1 });

  if (identity.answers.length != answerCursor.count()) {
    print('ERROR! ID: ' + identity._id + ' Legacy: ' + identity.answers.length + ' Radix: ' + answerCursor.count());
  }
});

Run after all imports are complete :: ensure related choice answers are added.

var cursor = db.getCollection('question').aggregate([
    { $match: { questionType: 'related-choice-single', relatedChoices: {$exists: true} } },
    { $project: { relatedChoices: 1 } },
    { $unwind: '$relatedChoices' },
    { $group: { 
      _id: null,
      choices: { $addToSet: '$relatedChoices'},
      mapped: { $addToSet: { question: '$_id', value: '$relatedChoices' } },
    } }
]);
if (cursor.hasNext()) {
  var doc = cursor.next();

  var map = {};
  doc.mapped.forEach(function(item) {
    var coerced = item.value.valueOf();
    if (false === map.hasOwnProperty(coerced)) {
      map[coerced] = [];
    }
    map[coerced].push(item.question);
  });

  var answerCursor = db.getCollection('identity-answer').find({
    'identity._type': 'identity-account',
    value: { $in: doc.choices },
  });
  var total = answerCursor.count();
  print(total + ' total answers found.');
  var processed = 0;


  answerCursor.forEach(function(answer) {
    delete answer._id;
    delete answer.legacy;
    var choiceId = answer.value.valueOf();
    if (true === map.hasOwnProperty(choiceId)) {
      map[choiceId].forEach(function(question) {
        answer.question = question;
        answer.wasAddedFromScript = true;
        db.getCollection('identity-answer').update({
          'identity._id': answer.identity._id,
          question: answer.question,
        }, {
          $setOnInsert: answer
        }, { upsert: true })
      });
    }
    processed++;
    print('Processed ' + processed + ' of ' + total + ' answers');
  });
  
}
print('Done!');
;