From be5f6e6de112e14eae4f09e3198e6a3e04918292 Mon Sep 17 00:00:00 2001 From: Hayden Chudy Date: Thu, 31 Mar 2016 21:41:33 -0400 Subject: [PATCH 1/4] Add venv and IDEA to .gitignore. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 6c98098..4630716 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.py[cod] +.env # C extensions *.so @@ -38,3 +39,7 @@ nosetests.xml *.db docs/_* + +# IntelliJ Metadata +*.iml +*.idea From 6f01b89b4400e428b0199db02b9c96049e157017 Mon Sep 17 00:00:00 2001 From: Hayden Chudy Date: Thu, 31 Mar 2016 21:42:03 -0400 Subject: [PATCH 2/4] Minimal requirements.txt needed for test suite. --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2825224 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +nose==1.3.7 +peewee==2.8.0 +termcolor==1.1.0 From 0cf90b45d7c9d23470492803a88a5214a1b3796d Mon Sep 17 00:00:00 2001 From: Hayden Chudy Date: Thu, 31 Mar 2016 21:44:29 -0400 Subject: [PATCH 3/4] Unit tests for out of order migrations. Right now, test_migrate_missing and test_migrate_missing_prevents_down fail. Added a test for arnold up 0 because I could. --- tests/__init__.py | 103 ++++++++++++++++-- .../migrations/003_migrate_missing.py | 19 ++++ tests/assets/002_missing_migration.py | 20 ++++ 3 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 tests/arnold_config/migrations/003_migrate_missing.py create mode 100644 tests/assets/002_missing_migration.py diff --git a/tests/__init__.py b/tests/__init__.py index f42f332..985a930 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -17,6 +17,16 @@ class BasicModel(Model): pass +class OutOfOrderMigration(Model): + class Meta: + database = db + + +class MissingMigration(Model): + class Meta: + database = db + + class TestMigrationFunctions(unittest.TestCase): def setUp(self): self.model = Migration @@ -27,13 +37,23 @@ def setUp(self): self.good_migration = "001_initial" self.bad_migration = "bad" + self.missing_migration = "002_missing_migration" + self.out_of_order_migration = "003_migrate_missing" + self.missing_migration_filename = self.missing_migration + '.py' def tearDown(self): self.model.drop_table() BasicModel.drop_table(fail_silently=True) + OutOfOrderMigration.drop_table(fail_silently=True) + MissingMigration.drop_table(fail_silently=True) if os.path.exists('./test_config'): shutil.rmtree('./test_config') + missing_migration_path = './arnold_config/migrations/{}'.format( + self.missing_migration_filename) + if os.path.exists(missing_migration_path): + os.unlink(missing_migration_path) + def test_setup_table(self): """Ensure that the Migration table will be setup properly""" # Drop the table if it exists, as we are creating it later @@ -43,19 +63,19 @@ def test_setup_table(self): Terminator(args) self.assertEqual(self.model.table_exists(), True) - def do_good_migration_up(self): + def do_good_migration_up(self, table_name="basicmodel"): """A utility to perform a successfull upwards migration""" args = parse_args(['up', '1']) termi = Terminator(args) termi.perform_migrations('up') - self.assertTrue("basicmodel" in db.get_tables()) + self.assertTrue(table_name in db.get_tables()) - def do_good_migration_down(self): + def do_good_migration_down(self, table_name="basicmodel"): """A utility to perform a successfull downwards migration""" args = parse_args(['down', '1']) termi = Terminator(args) termi.perform_migrations('down') - self.assertFalse("basicmodel" in db.get_tables()) + self.assertFalse(table_name in db.get_tables()) def test_perform_single_migration(self): """A simple test of _perform_single_migration""" @@ -63,7 +83,9 @@ def test_perform_single_migration(self): def test_perform_single_migration_already_migrated(self): """Run migration twice, second time should return False""" + # technically we run it 3 times self.do_good_migration_up() + self.do_good_migration_up("outofordermigration") args = parse_args(['up', '1']) termi = Terminator(args) @@ -95,15 +117,11 @@ def test_perform_single_migration_adds_deletes_row(self): """Make sure that the migration rows are added/deleted""" self.do_good_migration_up() - self.assertTrue(self.model.select().where( - self.model.migration == self.good_migration - ).limit(1).exists()) + self.assertTrue(self._migration_row_exists(self.good_migration)) self.do_good_migration_down() - self.assertFalse(self.model.select().where( - self.model.migration == self.good_migration - ).limit(1).exists()) + self.assertFalse(self._migration_row_exists(self.good_migration)) def test_with_fake_argument_returns_true_no_table(self): """If we pass fake, return true, but don't create the model table""" @@ -126,5 +144,70 @@ def test_init_creates_folders(self): self.assertTrue(os.path.isfile('./test_config/__init__.py')) self.assertTrue(os.path.isfile('./test_config/migrations/__init__.py')) + def test_out_of_order_migrations(self): + """If we don't pass --migrate-missing, don't migrate out of order.""" + # set up the state where an out of order migration can happen + args = parse_args(['up', '0']) + termi = Terminator(args) + termi.perform_migrations('up') + + def _assertions(): + self.assertTrue('outofordermigration' in db.get_tables()) + self.assertFalse('missingmigration' in db.get_tables()) + self.assertTrue(self._migration_row_exists( + self.out_of_order_migration)) + self.assertFalse(self._migration_row_exists(self.missing_migration)) + # assert that the setup succeeded + _assertions() + + # move the migration so we're in a proper missing state + shutil.copy('./assets/002_missing_migration.py', + './arnold_config/migrations/') + termi = Terminator(args) + termi.perform_migrations('up') + + # now that we've migrated, make sure we didn't add that table. + _assertions() + + def test_migrate_missing(self): + """If we pass --migrate-missing, run out of order migrations""" + self.test_out_of_order_migrations() + + # now let's ensure we add the table. + args = parse_args(['up', '0', '--migrate-missing']) + termi = Terminator(args) + termi.perform_migrations('up') + + self.assertTrue('outofordermigration' in db.get_tables()) + self.assertTrue('missingmigration' in db.get_tables()) + + self.assertTrue(self._migration_row_exists(self.out_of_order_migration)) + self.assertTrue(self._migration_row_exists(self.missing_migration)) + + def test_migrate_missing_prevents_down(self): + """Prevent going down with --migrate-missing.""" + self.do_good_migration_up() + args = parse_args(['down', '0']) + termi = Terminator(args) + # manually set this because the parser doesn't support it + termi.migrate_missing = True + self.assertFalse(termi.perform_migrations('down')) + + def test_migrate_0_migrates_all(self): + """Up 0 should migrate everything.""" + args = parse_args(['up', '0']) + termi = Terminator(args) + termi.perform_migrations('up') + + self.assertTrue(self._migration_row_exists(self.good_migration)) + self.assertTrue(self._migration_row_exists(self.out_of_order_migration)) + + def _migration_row_exists(self, migration_name): + """Assert a row exists in the migrations table with this name.""" + return self.model.select().where( + self.model.migration == migration_name + ).limit(1).exists() + + if __name__ == '__main__': unittest.main() diff --git a/tests/arnold_config/migrations/003_migrate_missing.py b/tests/arnold_config/migrations/003_migrate_missing.py new file mode 100644 index 0000000..d6d33fd --- /dev/null +++ b/tests/arnold_config/migrations/003_migrate_missing.py @@ -0,0 +1,19 @@ +from peewee import SqliteDatabase, Model, PrimaryKeyField + +from .. import database as db + + +class OutOfOrderMigration(Model): + """The migration model used to track migration status""" + id = PrimaryKeyField() + + class Meta: + database = db + + +def up(): + OutOfOrderMigration.create_table(fail_silently=True) + + +def down(): + OutOfOrderMigration.drop_table() diff --git a/tests/assets/002_missing_migration.py b/tests/assets/002_missing_migration.py new file mode 100644 index 0000000..126d34a --- /dev/null +++ b/tests/assets/002_missing_migration.py @@ -0,0 +1,20 @@ +from peewee import SqliteDatabase, Model, PrimaryKeyField + +from .. import database as db + + +class MissingMigration(Model): + """The migration model used to track migration status""" + id = PrimaryKeyField() + + class Meta: + database = db + + +def up(): + MissingMigration.create_table(fail_silently=True) + + +def down(): + MissingMigration.drop_table() + From d2748ed32712150764b0eec00b6e650a40de5dcc Mon Sep 17 00:00:00 2001 From: Hayden Chudy Date: Thu, 31 Mar 2016 22:35:36 -0400 Subject: [PATCH 4/4] Adds support for a new --migrate-missing flag for up. This flag allows running migrations out of order. It operates by grabbing the set of available migrations, and the set of already run migrations and taking their complement. That set of migrations is then run. This flag is only available on `up`, it makes no sense on `down`. **WARNING:** If you key your migrations numerically, this could be dangerous as it could result in migrations with the same prefix being run in an unspecified order. If you key by date, you should be fine. Ideally, write migrations that are non-destructive and non-dependent and you'll bee 100% fine. Also adds a few straggling tests. Such as ensuring migrations are run in the right order, with or without the new flag. New test to ensure latest migration is always based on name. Docs for the new feature and a warning on the arg help. --- README.md | 1 + arnold/__init__.py | 39 ++++++++++++++---- docs/configuration.rst | 12 +++++- tests/__init__.py | 41 ++++++++++++++++--- tests/assets/004_ensure_ordering_migration.py | 20 +++++++++ 5 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 tests/assets/004_ensure_ordering_migration.py diff --git a/README.md b/README.md index 4ba3272..4f1a0df 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Arnold accepts a number of configuration options to the commands. * --folder - The folder to use for configration/migrations. * --fake - Add the row to the database without running the migration. +* --migrate-missing - Run every migration that's missing, even if it's out of order. (WARNING: This can lead to data loss in certain situations. See [here](http://arnold.readthedocs.org/en/latest/configuration.html) for more info.) The `__init__.py` file inside the configuration folder holds the database value. This should be peewee database value. Here is an example `__init__.py` file: diff --git a/arnold/__init__.py b/arnold/__init__.py index cab93e5..bb82890 100644 --- a/arnold/__init__.py +++ b/arnold/__init__.py @@ -17,6 +17,7 @@ def __init__(self, args): self.fake = getattr(args, 'fake', False) self.count = getattr(args, 'count', 0) self.folder = getattr(args, 'folder', None) + self.migrate_missing = getattr(args, 'migrate_missing', False) self.prepare_config() self.database = self.config.database @@ -108,6 +109,10 @@ def get_latest_migration(self): self.model.migration.desc() ).first() + def get_run_migrations(self): + """Get the full set of migrations already run.""" + return self.model.select(self.model.migration) + def perform_migrations(self, direction): """ Find the migration if it is passed in and call the up or down method as @@ -128,7 +133,22 @@ def perform_migrations(self, direction): latest_migration = self.get_latest_migration() - if latest_migration: + if not latest_migration and self.direction == 'down': + print("Nothing to go {0}.".format( + colored(self.direction, "magenta")) + ) + return False + + if self.migrate_missing: + if self.direction == "down": + print("You cannot go {0} with missing migrations!".format( + colored(self.direction, "magenta") + )) + return False + run_migration_names = map(lambda model: model.migration, + self.get_run_migrations()) + filenames = set(filenames) - set(run_migration_names) + elif latest_migration: migration_index = filenames.index(latest_migration.migration) if migration_index == len(filenames) - 1 and \ @@ -142,18 +162,16 @@ def perform_migrations(self, direction): start = migration_index + 1 else: start = migration_index - if not latest_migration and self.direction == 'down': - print("Nothing to go {0}.".format( - colored(self.direction, "magenta")) - ) - return False if self.count == 0: end = len(filenames) else: end = start + self.count - migrations_to_complete = filenames[start:end] + if self.migrate_missing: + migrations_to_complete = filenames + else: + migrations_to_complete = filenames[start:end] if self.count > len(migrations_to_complete): print( @@ -228,6 +246,13 @@ def parse_args(args): '--fake', type=bool, default=False, help='Fake the migration.' ) + up_cmd.add_argument( + '--migrate-missing', default=False, dest="migrate_missing", + help="Automatically migrate any migrations that were not already run. " + "CAN BE DANGEROUS!", + action="store_true" + ) + down_cmd = subparsers.add_parser('down', help='Migrate down.') down_cmd.set_defaults(func=down) down_cmd.add_argument( diff --git a/docs/configuration.rst b/docs/configuration.rst index fa4dc3f..4f92dba 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -5,8 +5,18 @@ Arnold accepts a number of configuration options to the commands. * --folder - The folder to use for configration/migrations. * --fake - Add the row to the database without running the migration. +* --migrate-missing - Run every migration that's missing, even if it's out of order. -The `__init__.py` file inside the configuration folder holds the database value. This should be peewee database value. Here is an example `__init__.py` file: :: +.. warning:: + + The ``--migrate-missing`` option can lead to data corruption or loss depending + on how you write and name your migrations. If you prefix your migrations with a + simple numeric string, there is a possibility they may not be run in the order + you expect. If you prefix your migrations with the current date and time, you + are much safer. However, if you write non-destructive migrations that don't + heavily depend on each other (or at least other, recent ones), you'll be fine. + +The `__init__.py` file inside the configuration folder holds the database value. This should be a peewee database value. Here is an example `__init__.py` file: :: from peewee import SqliteDatabase diff --git a/tests/__init__.py b/tests/__init__.py index 985a930..0fdadf7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -39,7 +39,7 @@ def setUp(self): self.bad_migration = "bad" self.missing_migration = "002_missing_migration" self.out_of_order_migration = "003_migrate_missing" - self.missing_migration_filename = self.missing_migration + '.py' + self.ensure_ordering_migration = "004_ensure_ordering_migration" def tearDown(self): self.model.drop_table() @@ -49,11 +49,16 @@ def tearDown(self): if os.path.exists('./test_config'): shutil.rmtree('./test_config') - missing_migration_path = './arnold_config/migrations/{}'.format( - self.missing_migration_filename) + missing_migration_path = './arnold_config/migrations/{}.py'.format( + self.missing_migration) if os.path.exists(missing_migration_path): os.unlink(missing_migration_path) + ordering_migration_path = './arnold_config/migrations/{}.py'.format( + self.ensure_ordering_migration) + if os.path.exists(ordering_migration_path): + os.unlink(ordering_migration_path) + def test_setup_table(self): """Ensure that the Migration table will be setup properly""" # Drop the table if it exists, as we are creating it later @@ -161,8 +166,7 @@ def _assertions(): _assertions() # move the migration so we're in a proper missing state - shutil.copy('./assets/002_missing_migration.py', - './arnold_config/migrations/') + self._copy_asset(self.missing_migration) termi = Terminator(args) termi.perform_migrations('up') @@ -183,6 +187,9 @@ def test_migrate_missing(self): self.assertTrue(self._migration_row_exists(self.out_of_order_migration)) self.assertTrue(self._migration_row_exists(self.missing_migration)) + # the latest migration should still be the last one via name + self.assertEqual(termi.get_latest_migration().migration, + self.out_of_order_migration) def test_migrate_missing_prevents_down(self): """Prevent going down with --migrate-missing.""" @@ -202,6 +209,30 @@ def test_migrate_0_migrates_all(self): self.assertTrue(self._migration_row_exists(self.good_migration)) self.assertTrue(self._migration_row_exists(self.out_of_order_migration)) + def test_migrations_run_in_the_right_order(self): + """Ensure migrations always run in the right order.""" + def _test(migrate_missing=False): + """Simple wrapper since the test is the same with or without the + flag.""" + args = ['up', '0'] + self._copy_asset(self.ensure_ordering_migration) + + if migrate_missing: + self._copy_asset(self.missing_migration) + args.append('--migrate-missing') + + termi = Terminator(parse_args(args)) + termi.perform_migrations('up') + + self.assertTrue('outofordermigration' in db.get_tables()) + self.assertTrue(OutOfOrderMigration.select().first()) + _test() + _test(migrate_missing=True) + + def _copy_asset(self, migration_name): + return shutil.copy('./assets/{}.py'.format(migration_name), + './arnold_config/migrations/') + def _migration_row_exists(self, migration_name): """Assert a row exists in the migrations table with this name.""" return self.model.select().where( diff --git a/tests/assets/004_ensure_ordering_migration.py b/tests/assets/004_ensure_ordering_migration.py new file mode 100644 index 0000000..68e52c3 --- /dev/null +++ b/tests/assets/004_ensure_ordering_migration.py @@ -0,0 +1,20 @@ +from peewee import SqliteDatabase, Model, PrimaryKeyField + +from .. import database as db + + +class OutOfOrderMigration(Model): + """The migration model used to track migration status""" + id = PrimaryKeyField() + + class Meta: + database = db + + +def up(): + OutOfOrderMigration.insert().execute() + + +def down(): + OutOfOrderMigration.delete().execute() +