diff --git a/.gitignore b/.gitignore index 6c98098..4630716 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.py[cod] +.env # C extensions *.so @@ -38,3 +39,7 @@ nosetests.xml *.db docs/_* + +# IntelliJ Metadata +*.iml +*.idea diff --git a/README.md b/README.md index 4ba3272..4f1a0df 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Arnold accepts a number of configuration options to the commands. * --folder - The folder to use for configration/migrations. * --fake - Add the row to the database without running the migration. +* --migrate-missing - Run every migration that's missing, even if it's out of order. (WARNING: This can lead to data loss in certain situations. See [here](http://arnold.readthedocs.org/en/latest/configuration.html) for more info.) The `__init__.py` file inside the configuration folder holds the database value. This should be peewee database value. Here is an example `__init__.py` file: diff --git a/arnold/__init__.py b/arnold/__init__.py index cab93e5..bb82890 100644 --- a/arnold/__init__.py +++ b/arnold/__init__.py @@ -17,6 +17,7 @@ def __init__(self, args): self.fake = getattr(args, 'fake', False) self.count = getattr(args, 'count', 0) self.folder = getattr(args, 'folder', None) + self.migrate_missing = getattr(args, 'migrate_missing', False) self.prepare_config() self.database = self.config.database @@ -108,6 +109,10 @@ def get_latest_migration(self): self.model.migration.desc() ).first() + def get_run_migrations(self): + """Get the full set of migrations already run.""" + return self.model.select(self.model.migration) + def perform_migrations(self, direction): """ Find the migration if it is passed in and call the up or down method as @@ -128,7 +133,22 @@ def perform_migrations(self, direction): latest_migration = self.get_latest_migration() - if latest_migration: + if not latest_migration and self.direction == 'down': + print("Nothing to go {0}.".format( + colored(self.direction, "magenta")) + ) + return False + + if self.migrate_missing: + if self.direction == "down": + print("You cannot go {0} with missing migrations!".format( + colored(self.direction, "magenta") + )) + return False + run_migration_names = map(lambda model: model.migration, + self.get_run_migrations()) + filenames = set(filenames) - set(run_migration_names) + elif latest_migration: migration_index = filenames.index(latest_migration.migration) if migration_index == len(filenames) - 1 and \ @@ -142,18 +162,16 @@ def perform_migrations(self, direction): start = migration_index + 1 else: start = migration_index - if not latest_migration and self.direction == 'down': - print("Nothing to go {0}.".format( - colored(self.direction, "magenta")) - ) - return False if self.count == 0: end = len(filenames) else: end = start + self.count - migrations_to_complete = filenames[start:end] + if self.migrate_missing: + migrations_to_complete = filenames + else: + migrations_to_complete = filenames[start:end] if self.count > len(migrations_to_complete): print( @@ -228,6 +246,13 @@ def parse_args(args): '--fake', type=bool, default=False, help='Fake the migration.' ) + up_cmd.add_argument( + '--migrate-missing', default=False, dest="migrate_missing", + help="Automatically migrate any migrations that were not already run. " + "CAN BE DANGEROUS!", + action="store_true" + ) + down_cmd = subparsers.add_parser('down', help='Migrate down.') down_cmd.set_defaults(func=down) down_cmd.add_argument( diff --git a/docs/configuration.rst b/docs/configuration.rst index fa4dc3f..4f92dba 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -5,8 +5,18 @@ Arnold accepts a number of configuration options to the commands. * --folder - The folder to use for configration/migrations. * --fake - Add the row to the database without running the migration. +* --migrate-missing - Run every migration that's missing, even if it's out of order. -The `__init__.py` file inside the configuration folder holds the database value. This should be peewee database value. Here is an example `__init__.py` file: :: +.. warning:: + + The ``--migrate-missing`` option can lead to data corruption or loss depending + on how you write and name your migrations. If you prefix your migrations with a + simple numeric string, there is a possibility they may not be run in the order + you expect. If you prefix your migrations with the current date and time, you + are much safer. However, if you write non-destructive migrations that don't + heavily depend on each other (or at least other, recent ones), you'll be fine. + +The `__init__.py` file inside the configuration folder holds the database value. This should be a peewee database value. Here is an example `__init__.py` file: :: from peewee import SqliteDatabase diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2825224 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +nose==1.3.7 +peewee==2.8.0 +termcolor==1.1.0 diff --git a/tests/__init__.py b/tests/__init__.py index f42f332..0fdadf7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -17,6 +17,16 @@ class BasicModel(Model): pass +class OutOfOrderMigration(Model): + class Meta: + database = db + + +class MissingMigration(Model): + class Meta: + database = db + + class TestMigrationFunctions(unittest.TestCase): def setUp(self): self.model = Migration @@ -27,13 +37,28 @@ def setUp(self): self.good_migration = "001_initial" self.bad_migration = "bad" + self.missing_migration = "002_missing_migration" + self.out_of_order_migration = "003_migrate_missing" + self.ensure_ordering_migration = "004_ensure_ordering_migration" def tearDown(self): self.model.drop_table() BasicModel.drop_table(fail_silently=True) + OutOfOrderMigration.drop_table(fail_silently=True) + MissingMigration.drop_table(fail_silently=True) if os.path.exists('./test_config'): shutil.rmtree('./test_config') + missing_migration_path = './arnold_config/migrations/{}.py'.format( + self.missing_migration) + if os.path.exists(missing_migration_path): + os.unlink(missing_migration_path) + + ordering_migration_path = './arnold_config/migrations/{}.py'.format( + self.ensure_ordering_migration) + if os.path.exists(ordering_migration_path): + os.unlink(ordering_migration_path) + def test_setup_table(self): """Ensure that the Migration table will be setup properly""" # Drop the table if it exists, as we are creating it later @@ -43,19 +68,19 @@ def test_setup_table(self): Terminator(args) self.assertEqual(self.model.table_exists(), True) - def do_good_migration_up(self): + def do_good_migration_up(self, table_name="basicmodel"): """A utility to perform a successfull upwards migration""" args = parse_args(['up', '1']) termi = Terminator(args) termi.perform_migrations('up') - self.assertTrue("basicmodel" in db.get_tables()) + self.assertTrue(table_name in db.get_tables()) - def do_good_migration_down(self): + def do_good_migration_down(self, table_name="basicmodel"): """A utility to perform a successfull downwards migration""" args = parse_args(['down', '1']) termi = Terminator(args) termi.perform_migrations('down') - self.assertFalse("basicmodel" in db.get_tables()) + self.assertFalse(table_name in db.get_tables()) def test_perform_single_migration(self): """A simple test of _perform_single_migration""" @@ -63,7 +88,9 @@ def test_perform_single_migration(self): def test_perform_single_migration_already_migrated(self): """Run migration twice, second time should return False""" + # technically we run it 3 times self.do_good_migration_up() + self.do_good_migration_up("outofordermigration") args = parse_args(['up', '1']) termi = Terminator(args) @@ -95,15 +122,11 @@ def test_perform_single_migration_adds_deletes_row(self): """Make sure that the migration rows are added/deleted""" self.do_good_migration_up() - self.assertTrue(self.model.select().where( - self.model.migration == self.good_migration - ).limit(1).exists()) + self.assertTrue(self._migration_row_exists(self.good_migration)) self.do_good_migration_down() - self.assertFalse(self.model.select().where( - self.model.migration == self.good_migration - ).limit(1).exists()) + self.assertFalse(self._migration_row_exists(self.good_migration)) def test_with_fake_argument_returns_true_no_table(self): """If we pass fake, return true, but don't create the model table""" @@ -126,5 +149,96 @@ def test_init_creates_folders(self): self.assertTrue(os.path.isfile('./test_config/__init__.py')) self.assertTrue(os.path.isfile('./test_config/migrations/__init__.py')) + def test_out_of_order_migrations(self): + """If we don't pass --migrate-missing, don't migrate out of order.""" + # set up the state where an out of order migration can happen + args = parse_args(['up', '0']) + termi = Terminator(args) + termi.perform_migrations('up') + + def _assertions(): + self.assertTrue('outofordermigration' in db.get_tables()) + self.assertFalse('missingmigration' in db.get_tables()) + self.assertTrue(self._migration_row_exists( + self.out_of_order_migration)) + self.assertFalse(self._migration_row_exists(self.missing_migration)) + # assert that the setup succeeded + _assertions() + + # move the migration so we're in a proper missing state + self._copy_asset(self.missing_migration) + termi = Terminator(args) + termi.perform_migrations('up') + + # now that we've migrated, make sure we didn't add that table. + _assertions() + + def test_migrate_missing(self): + """If we pass --migrate-missing, run out of order migrations""" + self.test_out_of_order_migrations() + + # now let's ensure we add the table. + args = parse_args(['up', '0', '--migrate-missing']) + termi = Terminator(args) + termi.perform_migrations('up') + + self.assertTrue('outofordermigration' in db.get_tables()) + self.assertTrue('missingmigration' in db.get_tables()) + + self.assertTrue(self._migration_row_exists(self.out_of_order_migration)) + self.assertTrue(self._migration_row_exists(self.missing_migration)) + # the latest migration should still be the last one via name + self.assertEqual(termi.get_latest_migration().migration, + self.out_of_order_migration) + + def test_migrate_missing_prevents_down(self): + """Prevent going down with --migrate-missing.""" + self.do_good_migration_up() + args = parse_args(['down', '0']) + termi = Terminator(args) + # manually set this because the parser doesn't support it + termi.migrate_missing = True + self.assertFalse(termi.perform_migrations('down')) + + def test_migrate_0_migrates_all(self): + """Up 0 should migrate everything.""" + args = parse_args(['up', '0']) + termi = Terminator(args) + termi.perform_migrations('up') + + self.assertTrue(self._migration_row_exists(self.good_migration)) + self.assertTrue(self._migration_row_exists(self.out_of_order_migration)) + + def test_migrations_run_in_the_right_order(self): + """Ensure migrations always run in the right order.""" + def _test(migrate_missing=False): + """Simple wrapper since the test is the same with or without the + flag.""" + args = ['up', '0'] + self._copy_asset(self.ensure_ordering_migration) + + if migrate_missing: + self._copy_asset(self.missing_migration) + args.append('--migrate-missing') + + termi = Terminator(parse_args(args)) + termi.perform_migrations('up') + + self.assertTrue('outofordermigration' in db.get_tables()) + self.assertTrue(OutOfOrderMigration.select().first()) + _test() + _test(migrate_missing=True) + + def _copy_asset(self, migration_name): + return shutil.copy('./assets/{}.py'.format(migration_name), + './arnold_config/migrations/') + + def _migration_row_exists(self, migration_name): + """Assert a row exists in the migrations table with this name.""" + return self.model.select().where( + self.model.migration == migration_name + ).limit(1).exists() + + if __name__ == '__main__': unittest.main() diff --git a/tests/arnold_config/migrations/003_migrate_missing.py b/tests/arnold_config/migrations/003_migrate_missing.py new file mode 100644 index 0000000..d6d33fd --- /dev/null +++ b/tests/arnold_config/migrations/003_migrate_missing.py @@ -0,0 +1,19 @@ +from peewee import SqliteDatabase, Model, PrimaryKeyField + +from .. import database as db + + +class OutOfOrderMigration(Model): + """The migration model used to track migration status""" + id = PrimaryKeyField() + + class Meta: + database = db + + +def up(): + OutOfOrderMigration.create_table(fail_silently=True) + + +def down(): + OutOfOrderMigration.drop_table() diff --git a/tests/assets/002_missing_migration.py b/tests/assets/002_missing_migration.py new file mode 100644 index 0000000..126d34a --- /dev/null +++ b/tests/assets/002_missing_migration.py @@ -0,0 +1,20 @@ +from peewee import SqliteDatabase, Model, PrimaryKeyField + +from .. import database as db + + +class MissingMigration(Model): + """The migration model used to track migration status""" + id = PrimaryKeyField() + + class Meta: + database = db + + +def up(): + MissingMigration.create_table(fail_silently=True) + + +def down(): + MissingMigration.drop_table() + diff --git a/tests/assets/004_ensure_ordering_migration.py b/tests/assets/004_ensure_ordering_migration.py new file mode 100644 index 0000000..68e52c3 --- /dev/null +++ b/tests/assets/004_ensure_ordering_migration.py @@ -0,0 +1,20 @@ +from peewee import SqliteDatabase, Model, PrimaryKeyField + +from .. import database as db + + +class OutOfOrderMigration(Model): + """The migration model used to track migration status""" + id = PrimaryKeyField() + + class Meta: + database = db + + +def up(): + OutOfOrderMigration.insert().execute() + + +def down(): + OutOfOrderMigration.delete().execute() +