Add support for schema= in migration tooling.

Author: coleifer

CHANGELOG.md

@@ -33,6 +33,10 @@ Improvements:
   `get_primary_keys()`, `get_foreign_keys()`, `get_views()`) with no `schema`
   now follows the search path via `current_schema()` instead of assuming
   `public`, matching MySQL's `DATABASE()` behavior.
+* Add `Runner(db, schema=...)` and `pwmigrate --schema` for running
+  migrations against a specific schema. The history table lives in that
+  schema, so each schema tracks its own applied set and one set of migration
+  files can be run against any number of schemas.
 * Add `SchemaMigrator(db, schema=...)`, which qualifies every table name w/the
   given schema (or database on MySQL). Not supported for SQLite, as the
   table-rebuild rewrites DDL straight from `sqlite_master`.

docs/peewee/db_tools.rst

@@ -1010,6 +1010,7 @@ Recognized keys:
 * ``database`` - database spec (dotted path, url, or sqlite filename)
 * ``directory`` - migrations directory
 * ``models`` - models module, read by ``diff``, ``initial`` and ``generate``
+* ``schema`` - schema containing the tables to be migrated
 * ``table`` - history table name
 
 The file is read from the working directory only. A different config
@@ -1111,7 +1112,17 @@ Generate a migration from a diff:
    if diff:
        runner.create('add karma', body=template(diff))
 
-.. class:: Runner(database, directory='migrations', table_name='schema_migration')
+.. class:: Runner(database, directory='migrations', table_name='schema_migration', schema=None)
+
+   :param str schema: schema containing the tables to be migrated, passed to
+       the :class:`SchemaMigrator`. The history table lives in the same
+       schema, so each schema tracks its own applied set and one set of
+       migration files can be run against any number of schemas
+       (``pwmigrate up -s tenant_a``, ``pwmigrate up -s tenant_b``).
+
+       When adopting ``schema=`` on a deployment whose history predates it,
+       the runner will find no history there and consider every migration
+       pending. Backfill with ``fake`` first.
 
    .. method:: up(target=None)
 

playhouse/migrations.py

@@ -89,18 +89,22 @@ def load(self):
 
 class Runner(object):
     def __init__(self, database, directory='migrations',
-                 table_name='schema_migration'):
+                 table_name='schema_migration', schema=None):
         self.database = database
         self.directory = directory
         self.table_name = table_name
-        self.migrator = SchemaMigrator.from_database(database)
+        self.schema = schema
+        self.migrator = SchemaMigrator.from_database(database, schema=schema)
 
+        # History lives in the schema it describes, so each schema tracks
+        # its own applied set.
         class History(database.Model):
             name = CharField(unique=True)
             applied = DateTimeField(default=datetime.datetime.now)
             class Meta:
                 legacy_table_names = False
                 table_name = self.table_name
+                schema = self.schema
 
         self.History = History
 
@@ -496,7 +500,7 @@ def template(diff):
                     for f in diff.add_columns if f.model._meta.schema)
     if qualified:
         todos.append('schema-qualified tables (%s): column/index '
-                     'operations are emitted unqualified. Qualify by hand' %
+                     'operations are emitted unqualified. Run with --schema' %
                      ', '.join(sorted(qualified)))
 
     up, down = [], []
@@ -713,7 +717,7 @@ def _read_config(path):
             key, _, value = line.partition('=')
             config[key.strip()] = value.strip()
     for key in sorted(set(config) - {'database', 'directory', 'models',
-                                     'table'}):
+                                     'schema', 'table'}):
         sys.stderr.write('warning: unknown key "%s" in %s\n' % (key, path))
     return config
 
@@ -735,6 +739,9 @@ def _parser(config):
     common.add_argument('-d', '--directory',
                         default=config.get('directory', 'migrations'),
                         help='migrations directory (default: migrations)')
+    common.add_argument('-s', '--schema',
+                        default=config.get('schema'),
+                        help='schema containing the tables to be migrated')
     common.add_argument('-t', '--table',
                         default=config.get('table', 'schema_migration'),
                         help='history table name')
@@ -823,10 +830,10 @@ def main(argv=None):
     database = None
     try:
         database = _resolve_database(args.database)
-        runner = Runner(database, args.directory, args.table)
+        runner = Runner(database, args.directory, args.table, args.schema)
         return args.func(runner, args) or 0
     except (MigrationError, DatabaseError, InterfaceError,
-            ImproperlyConfigured) as exc:
+            ImproperlyConfigured, ValueError) as exc:
         if args.verbose:
             traceback.print_exc()
         sys.stderr.write('error: %s\n' % exc)

tests/migrations.py

@@ -1731,6 +1731,72 @@ def run_cli(*args):
     return rc, out.getvalue(), err.getvalue()
 
 
+@requires_postgresql
+class TestRunnerSchema(DatabaseTestCase):
+    """One set of migration files runs against any number of schemas."""
+    target = 'runner_target'
+    decoy = 'runner_decoy'
+
+    def setUp(self):
+        super(TestRunnerSchema, self).setUp()
+        self.dir = tempfile.mkdtemp()
+        for schema in (self.target, self.decoy):
+            self.execute('DROP SCHEMA IF EXISTS %s CASCADE' % schema)
+            self.execute('CREATE SCHEMA %s' % schema)
+            self.execute('CREATE TABLE %s.person (id SERIAL PRIMARY KEY, '
+                         'first_name TEXT)' % schema)
+        with open(os.path.join(self.dir, '0001_notes.py'), 'w') as fh:
+            fh.write(add_column_mig('notes'))
+        self.runner = Runner(self.database, self.dir, schema=self.target)
+
+    def tearDown(self):
+        try:
+            shutil.rmtree(self.dir, ignore_errors=True)
+            for schema in (self.target, self.decoy):
+                self.execute('DROP SCHEMA IF EXISTS %s CASCADE' % schema)
+        finally:
+            super(TestRunnerSchema, self).tearDown()
+
+    def columns(self, schema):
+        return sorted(c.name for c in
+                      self.database.get_columns('person', schema))
+
+    def tables(self, schema):
+        return self.database.get_tables(schema)
+
+    def test_up_down(self):
+        self.assertEqual(self.runner.up(), ['0001_notes'])
+        self.assertEqual(self.columns(self.target),
+                         ['first_name', 'id', 'notes'])
+        self.assertEqual(self.columns(self.decoy), ['first_name', 'id'])
+        # History lives in the schema it describes.
+        self.assertTrue('schema_migration' in self.tables(self.target))
+        self.assertFalse('schema_migration' in self.tables(self.decoy))
+
+        self.assertEqual(self.runner.down(), ['0001_notes'])
+        self.assertEqual(self.columns(self.target), ['first_name', 'id'])
+
+    def test_per_schema_history(self):
+        self.runner.up()
+        other = Runner(self.database, self.dir, schema=self.decoy)
+        self.assertEqual(other.up(), ['0001_notes'])
+        self.assertEqual(self.columns(self.decoy),
+                         ['first_name', 'id', 'notes'])
+        self.assertEqual(sorted(self.runner.applied()), ['0001_notes'])
+        self.assertEqual(sorted(other.applied()), ['0001_notes'])
+
+        # Each schema reverts independently.
+        other.down()
+        self.assertEqual(self.columns(self.decoy), ['first_name', 'id'])
+        self.assertEqual(self.columns(self.target),
+                         ['first_name', 'id', 'notes'])
+
+    def test_fake_backfill(self):
+        self.assertEqual(self.runner.fake(), ['0001_notes'])
+        self.assertEqual(self.columns(self.target), ['first_name', 'id'])
+        self.assertEqual(self.runner.up(), [])
+
+
 class TestMigrationRunnerCLI(BaseTestCase):
     def setUp(self):
         super(TestMigrationRunnerCLI, self).setUp()