Add a config option to run all missing migrations (even those possibly out of order)

.gitignore

@@ -1,4 +1,5 @@
 *.py[cod]
+.env
 
 # C extensions
 *.so
@@ -38,3 +39,7 @@ nosetests.xml
 *.db
 
 docs/_*
+
+# IntelliJ Metadata
+*.iml
+*.idea

README.md

@@ -86,6 +86,7 @@ Arnold accepts a number of configuration options to the commands.
 
 * --folder - The folder to use for configration/migrations.
 * --fake   - Add the row to the database without running the migration.
+* --migrate-missing - Run every migration that's missing, even if it's out of order. (WARNING: This can lead to data loss in certain situations. See [here](http://arnold.readthedocs.org/en/latest/configuration.html) for more info.)
 
 The `__init__.py` file inside the configuration folder holds the database value. This should be peewee database value. Here is an example `__init__.py` file:
 

arnold/__init__.py

@@ -17,6 +17,7 @@ def __init__(self, args):
         self.fake = getattr(args, 'fake', False)
         self.count = getattr(args, 'count', 0)
         self.folder = getattr(args, 'folder', None)
+        self.migrate_missing = getattr(args, 'migrate_missing', False)
 
         self.prepare_config()
         self.database = self.config.database
@@ -108,6 +109,10 @@ def get_latest_migration(self):
             self.model.migration.desc()
         ).first()
 
+    def get_run_migrations(self):
+        """Get the full set of migrations already run."""
+        return self.model.select(self.model.migration)
+
     def perform_migrations(self, direction):
         """
         Find the migration if it is passed in and call the up or down method as
@@ -128,7 +133,22 @@ def perform_migrations(self, direction):
 
         latest_migration = self.get_latest_migration()
 
-        if latest_migration:
+        if not latest_migration and self.direction == 'down':
+            print("Nothing to go {0}.".format(
+                    colored(self.direction, "magenta"))
+            )
+            return False
+
+        if self.migrate_missing:
+            if self.direction == "down":
+                print("You cannot go {0} with missing migrations!".format(
+                    colored(self.direction, "magenta")
+                ))
+                return False
+            run_migration_names = map(lambda model: model.migration,
+                                      self.get_run_migrations())
+            filenames = set(filenames) - set(run_migration_names)
+        elif latest_migration:
             migration_index = filenames.index(latest_migration.migration)
 
             if migration_index == len(filenames) - 1 and \
@@ -142,18 +162,16 @@ def perform_migrations(self, direction):
                 start = migration_index + 1
             else:
                 start = migration_index
-        if not latest_migration and self.direction == 'down':
-            print("Nothing to go {0}.".format(
-                colored(self.direction, "magenta"))
-            )
-            return False
 
         if self.count == 0:
             end = len(filenames)
         else:
             end = start + self.count
 
-        migrations_to_complete = filenames[start:end]
+        if self.migrate_missing:
+            migrations_to_complete = filenames
+        else:
+            migrations_to_complete = filenames[start:end]
 
         if self.count > len(migrations_to_complete):
             print(
@@ -228,6 +246,13 @@ def parse_args(args):
         '--fake', type=bool, default=False, help='Fake the migration.'
     )
 
+    up_cmd.add_argument(
+        '--migrate-missing', default=False, dest="migrate_missing",
+        help="Automatically migrate any migrations that were not already run. "
+             "CAN BE DANGEROUS!",
+        action="store_true"
+    )
+
     down_cmd = subparsers.add_parser('down', help='Migrate down.')
     down_cmd.set_defaults(func=down)
     down_cmd.add_argument(

docs/configuration.rst

@@ -5,8 +5,18 @@ Arnold accepts a number of configuration options to the commands.
 
 * --folder - The folder to use for configration/migrations.
 * --fake   - Add the row to the database without running the migration.
+* --migrate-missing - Run every migration that's missing, even if it's out of order.
 
-The `__init__.py` file inside the configuration folder holds the database value. This should be peewee database value. Here is an example `__init__.py` file: ::
+.. warning::
+
+   The ``--migrate-missing`` option can lead to data corruption or loss depending
+   on how you write and name your migrations. If you prefix your migrations with a
+   simple numeric string, there is a possibility they may not be run in the order
+   you expect. If you prefix your migrations with the current date and time, you
+   are much safer. However, if you write non-destructive migrations that don't
+   heavily depend on each other (or at least other, recent ones), you'll be fine.
+
+The `__init__.py` file inside the configuration folder holds the database value. This should be a peewee database value. Here is an example `__init__.py` file: ::
 
   from peewee import SqliteDatabase
 

requirements.txt

@@ -0,0 +1,3 @@
+nose==1.3.7
+peewee==2.8.0
+termcolor==1.1.0

tests/__init__.py

@@ -17,6 +17,16 @@ class BasicModel(Model):
     pass
 
 
+class OutOfOrderMigration(Model):
+    class Meta:
+        database = db
+
+
+class MissingMigration(Model):
+    class Meta:
+        database = db
+
+
 class TestMigrationFunctions(unittest.TestCase):
     def setUp(self):
         self.model = Migration
@@ -27,13 +37,28 @@ def setUp(self):
 
         self.good_migration = "001_initial"
         self.bad_migration = "bad"
+        self.missing_migration = "002_missing_migration"
+        self.out_of_order_migration = "003_migrate_missing"
+        self.ensure_ordering_migration = "004_ensure_ordering_migration"
 
     def tearDown(self):
         self.model.drop_table()
         BasicModel.drop_table(fail_silently=True)
+        OutOfOrderMigration.drop_table(fail_silently=True)
+        MissingMigration.drop_table(fail_silently=True)
         if os.path.exists('./test_config'):
             shutil.rmtree('./test_config')
 
+        missing_migration_path = './arnold_config/migrations/{}.py'.format(
+            self.missing_migration)
+        if os.path.exists(missing_migration_path):
+            os.unlink(missing_migration_path)
+
+        ordering_migration_path = './arnold_config/migrations/{}.py'.format(
+            self.ensure_ordering_migration)
+        if os.path.exists(ordering_migration_path):
+            os.unlink(ordering_migration_path)
+
     def test_setup_table(self):
         """Ensure that the Migration table will be setup properly"""
         # Drop the table if it exists, as we are creating it later
@@ -43,27 +68,29 @@ def test_setup_table(self):
         Terminator(args)
         self.assertEqual(self.model.table_exists(), True)
 
-    def do_good_migration_up(self):
+    def do_good_migration_up(self, table_name="basicmodel"):
         """A utility to perform a successfull upwards migration"""
         args = parse_args(['up', '1'])
         termi = Terminator(args)
         termi.perform_migrations('up')
-        self.assertTrue("basicmodel" in db.get_tables())
+        self.assertTrue(table_name in db.get_tables())
 
-    def do_good_migration_down(self):
+    def do_good_migration_down(self, table_name="basicmodel"):
         """A utility to perform a successfull downwards migration"""
         args = parse_args(['down', '1'])
         termi = Terminator(args)
         termi.perform_migrations('down')
-        self.assertFalse("basicmodel" in db.get_tables())
+        self.assertFalse(table_name in db.get_tables())
 
     def test_perform_single_migration(self):
         """A simple test of _perform_single_migration"""
         self.do_good_migration_up()
 
     def test_perform_single_migration_already_migrated(self):
         """Run migration twice, second time should return False"""
+        # technically we run it 3 times
         self.do_good_migration_up()
+        self.do_good_migration_up("outofordermigration")
 
         args = parse_args(['up', '1'])
         termi = Terminator(args)
@@ -95,15 +122,11 @@ def test_perform_single_migration_adds_deletes_row(self):
         """Make sure that the migration rows are added/deleted"""
         self.do_good_migration_up()
 
-        self.assertTrue(self.model.select().where(
-            self.model.migration == self.good_migration
-        ).limit(1).exists())
+        self.assertTrue(self._migration_row_exists(self.good_migration))
 
         self.do_good_migration_down()
 
-        self.assertFalse(self.model.select().where(
-            self.model.migration == self.good_migration
-        ).limit(1).exists())
+        self.assertFalse(self._migration_row_exists(self.good_migration))
 
     def test_with_fake_argument_returns_true_no_table(self):
         """If we pass fake, return true, but don't create the model table"""
@@ -126,5 +149,96 @@ def test_init_creates_folders(self):
         self.assertTrue(os.path.isfile('./test_config/__init__.py'))
         self.assertTrue(os.path.isfile('./test_config/migrations/__init__.py'))
 
+    def test_out_of_order_migrations(self):
+        """If we don't pass --migrate-missing, don't migrate out of order."""
+        # set up the state where an out of order migration can happen
+        args = parse_args(['up', '0'])
+        termi = Terminator(args)
+        termi.perform_migrations('up')
+
+        def _assertions():
+            self.assertTrue('outofordermigration' in db.get_tables())
+            self.assertFalse('missingmigration' in db.get_tables())
+            self.assertTrue(self._migration_row_exists(
+                    self.out_of_order_migration))
+            self.assertFalse(self._migration_row_exists(self.missing_migration))
+        # assert that the setup succeeded
+        _assertions()
+
+        # move the migration so we're in a proper missing state
+        self._copy_asset(self.missing_migration)
+        termi = Terminator(args)
+        termi.perform_migrations('up')
+
+        # now that we've migrated, make sure we didn't add that table.
+        _assertions()
+
+    def test_migrate_missing(self):
+        """If we pass --migrate-missing, run out of order migrations"""
+        self.test_out_of_order_migrations()
+
+        # now let's ensure we add the table.
+        args = parse_args(['up', '0', '--migrate-missing'])
+        termi = Terminator(args)
+        termi.perform_migrations('up')
+
+        self.assertTrue('outofordermigration' in db.get_tables())
+        self.assertTrue('missingmigration' in db.get_tables())
+
+        self.assertTrue(self._migration_row_exists(self.out_of_order_migration))
+        self.assertTrue(self._migration_row_exists(self.missing_migration))
+        # the latest migration should still be the last one via name
+        self.assertEqual(termi.get_latest_migration().migration,
+                         self.out_of_order_migration)
+
+    def test_migrate_missing_prevents_down(self):
+        """Prevent going down with --migrate-missing."""
+        self.do_good_migration_up()
+        args = parse_args(['down', '0'])
+        termi = Terminator(args)
+        # manually set this because the parser doesn't support it
+        termi.migrate_missing = True
+        self.assertFalse(termi.perform_migrations('down'))
+
+    def test_migrate_0_migrates_all(self):
+        """Up 0 should migrate everything."""
+        args = parse_args(['up', '0'])
+        termi = Terminator(args)
+        termi.perform_migrations('up')
+
+        self.assertTrue(self._migration_row_exists(self.good_migration))
+        self.assertTrue(self._migration_row_exists(self.out_of_order_migration))
+
+    def test_migrations_run_in_the_right_order(self):
+        """Ensure migrations always run in the right order."""
+        def _test(migrate_missing=False):
+            """Simple wrapper since the test is the same with or without the
+            flag."""
+            args = ['up', '0']
+            self._copy_asset(self.ensure_ordering_migration)
+
+            if migrate_missing:
+                self._copy_asset(self.missing_migration)
+                args.append('--migrate-missing')
+
+            termi = Terminator(parse_args(args))
+            termi.perform_migrations('up')
+
+            self.assertTrue('outofordermigration' in db.get_tables())
+            self.assertTrue(OutOfOrderMigration.select().first())
+        _test()
+        _test(migrate_missing=True)
+
+    def _copy_asset(self, migration_name):
+        return shutil.copy('./assets/{}.py'.format(migration_name),
+                    './arnold_config/migrations/')
+
+    def _migration_row_exists(self, migration_name):
+        """Assert a row exists in the migrations table with this name."""
+        return self.model.select().where(
+                self.model.migration == migration_name
+        ).limit(1).exists()
+
+
 if __name__ == '__main__':
     unittest.main()

tests/arnold_config/migrations/003_migrate_missing.py

@@ -0,0 +1,19 @@
+from peewee import SqliteDatabase, Model, PrimaryKeyField
+
+from .. import database as db
+
+
+class OutOfOrderMigration(Model):
+    """The migration model used to track migration status"""
+    id = PrimaryKeyField()
+
+    class Meta:
+        database = db
+
+
+def up():
+    OutOfOrderMigration.create_table(fail_silently=True)
+
+
+def down():
+    OutOfOrderMigration.drop_table()

tests/assets/002_missing_migration.py

@@ -0,0 +1,20 @@
+from peewee import SqliteDatabase, Model, PrimaryKeyField
+
+from .. import database as db
+
+
+class MissingMigration(Model):
+    """The migration model used to track migration status"""
+    id = PrimaryKeyField()
+
+    class Meta:
+        database = db
+
+
+def up():
+    MissingMigration.create_table(fail_silently=True)
+
+
+def down():
+    MissingMigration.drop_table()
+

tests/assets/004_ensure_ordering_migration.py

@@ -0,0 +1,20 @@
+from peewee import SqliteDatabase, Model, PrimaryKeyField
+
+from .. import database as db
+
+
+class OutOfOrderMigration(Model):
+    """The migration model used to track migration status"""
+    id = PrimaryKeyField()
+
+    class Meta:
+        database = db
+
+
+def up():
+    OutOfOrderMigration.insert().execute()
+
+
+def down():
+    OutOfOrderMigration.delete().execute()
+