Merge pull request #3 from Overseas-Student-Living/PLAT-928

Plat 928

Author: mattbennett

config/config.yaml

@@ -0,0 +1,2 @@
+DB_URIS:
+    "test": "mysql+mysqlconnector://root:password@dockermachine:3306/sqlalchemydiff"

dists/mysql-connector-python-2.0.4.zip

@@ -61,9 +61,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.0.1'
+version = '0.1.0'
 # The full version, including alpha/beta/rc tags.
-release = '0.0.1'
+release = '0.1.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/conf.py

@@ -77,6 +77,20 @@ them by looking at the ``errors`` dict on the ``result``:
             }
 
 
+You can also tell the comparer to ignore parts of the database.
+
+For example, to ignore the ``users`` table completely, and the ``line3`` column
+of the table ``addresses`` you would call ``compare`` like this:
+
+.. code-block:: Python
+
+    >>> result = compare(
+            uri_left,
+            uri_right,
+            ignores=['users', 'addresses.col.line3']
+        )
+
+
 If you wish to persist that dict to a JSON file, you can quickly do so
 by calling ``result.dump_errors()``.
 
@@ -91,6 +105,8 @@ Currently the library can detect the following differences:
 - Differences in **Foreign Keys** for a common table
 - Differences in **Indexes** for a common table
 - Differences in **Columns** for a common table
+- Ability to ignore a **whole table**
+- Ability to ignore **primary/foreign keys**, **indexes** and **columns**
 
 
 Installation

docs/source/index.rst

@@ -13,7 +13,7 @@
 
 setup(
     name='sqlalchemy-diff',
-    version='0.0.4',
+    version='0.1.0',
     description='Compare two database schemas using sqlalchemy.',
     long_description=readme,
     author='student.com',
@@ -22,21 +22,19 @@
     packages=find_packages(exclude=['docs', 'test', 'test.*']),
     install_requires=[
         "six==1.10.0",
-        "mock>=1.3.0",
-        "sqlalchemy-utils==0.31.2",
+        "mock>=2.0.0",
+        "sqlalchemy-utils==0.32.4",
+        "pyyaml==3.11",
     ],
     extras_require={
         'dev': [
             "mysql-connector-python==2.0.4",
-            "pytest==2.8.2",
+            "pytest==2.9.1",
         ],
         'docs': [
-            "Sphinx==1.3.1",
+            "sphinx==1.4.1",
         ],
     },
-    dependency_links=[
-        "dists/mysql-connector-python-2.0.4.zip",
-    ],
     zip_safe=True,
     license='Apache License, Version 2.0',
     classifiers=[

setup.py

@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
 from copy import deepcopy
 
-from .util import TablesInfo, DiffResult, InspectorFactory, CompareResult
+from .util import (
+    TablesInfo, DiffResult, InspectorFactory, CompareResult, IgnoreManager
+)
 
 
-def compare(left_uri, right_uri, ignore_tables=None):
+def compare(left_uri, right_uri, ignores=None, ignores_sep=None):
     """Compare two databases, given two URIs.
 
-    Compare two databases, given two URIs and a (possibly empty) set of
-    tables to ignore during the comparison.
+    Compare two databases, ignoring whatever is specified in `ignores`.
 
     The ``info`` dict has this structure::
 
@@ -63,26 +64,34 @@ def compare(left_uri, right_uri, ignore_tables=None):
 
     :param string left_uri: The URI for the first (left) database.
     :param string right_uri: The URI for the second (right) database.
-    :param set ignore_tables:
-        A set of string values to be excluded from both databases (if
-        present) when doing the comparison.  String matching is case
-        sensitive.
+    :param iterable ignores:
+        A list of strings in the format:
+          * `table-name`
+          * `table-name.identifier.name`
+
+        If a table name is specified, the whole table is excluded from
+        comparison.  If a complete clause is specified, then only the
+        specified element is excluded from comparison.  `identifier` is one
+        of (`col`, `pk`, `fk`, `idx`) and name is the name of the element
+        to be excluded from the comparison.
+    :param string ignores_sep:
+        Separator to be used to spilt the `ignores` clauses.
     :return:
         A :class:`~.util.CompareResult` object with ``info`` and
         ``errors`` dicts populated with the comparison result.
     """
-    if ignore_tables is None:
-        ignore_tables = set()
+    ignore_manager = IgnoreManager(ignores, separator=ignores_sep)
 
     left_inspector, right_inspector = _get_inspectors(left_uri, right_uri)
 
     tables_info = _get_tables_info(
-        left_inspector, right_inspector, ignore_tables)
+        left_inspector, right_inspector, ignore_manager.ignore_tables)
 
     info = _get_info_dict(left_uri, right_uri, tables_info)
 
     info['tables_data'] = _get_tables_data(
-        tables_info.common, left_inspector, right_inspector)
+        tables_info.common, left_inspector, right_inspector, ignore_manager
+    )
 
     errors = _compile_errors(info)
     result = _make_result(info, errors)
@@ -157,32 +166,53 @@ def _get_info_dict(left_uri, right_uri, tables_info):
     return info
 
 
-def _get_tables_data(tables_common, left_inspector, right_inspector):
+def _get_tables_data(
+    tables_common, left_inspector, right_inspector, ignore_manager
+):
     tables_data = {}
 
     for table_name in tables_common:
         table_data = _get_table_data(
-            left_inspector, right_inspector, table_name)
+            left_inspector, right_inspector, table_name, ignore_manager
+        )
         tables_data[table_name] = table_data
 
     return tables_data
 
 
-def _get_table_data(left_inspector, right_inspector, table_name):
+def _get_table_data(
+    left_inspector, right_inspector, table_name, ignore_manager
+):
     table_data = {}
 
     # foreign keys
     table_data['foreign_keys'] = _get_foreign_keys_info(
-        left_inspector, right_inspector, table_name)
+        left_inspector,
+        right_inspector,
+        table_name,
+        ignore_manager.get(table_name, 'fk')
+    )
 
     table_data['primary_keys'] = _get_primary_keys_info(
-        left_inspector, right_inspector, table_name)
+        left_inspector,
+        right_inspector,
+        table_name,
+        ignore_manager.get(table_name, 'pk')
+    )
 
     table_data['indexes'] = _get_indexes_info(
-        left_inspector, right_inspector, table_name)
+        left_inspector,
+        right_inspector,
+        table_name,
+        ignore_manager.get(table_name, 'idx')
+    )
 
     table_data['columns'] = _get_columns_info(
-        left_inspector, right_inspector, table_name)
+        left_inspector,
+        right_inspector,
+        table_name,
+        ignore_manager.get(table_name, 'col')
+    )
 
     return table_data
 
@@ -225,10 +255,15 @@ def _diff_dicts(left, right):
     )._asdict()
 
 
-def _get_foreign_keys_info(left_inspector, right_inspector, table_name):
+def _get_foreign_keys_info(
+    left_inspector, right_inspector, table_name, ignores
+):
     left_fk_list = _get_foreign_keys(left_inspector, table_name)
     right_fk_list = _get_foreign_keys(right_inspector, table_name)
 
+    left_fk_list = _discard_ignores_by_name(left_fk_list, ignores)
+    right_fk_list = _discard_ignores_by_name(right_fk_list, ignores)
+
     # process into dict
     left_fk = dict((elem['name'], elem) for elem in left_fk_list)
     right_fk = dict((elem['name'], elem) for elem in right_fk_list)
@@ -240,10 +275,15 @@ def _get_foreign_keys(inspector, table_name):
     return inspector.get_foreign_keys(table_name)
 
 
-def _get_primary_keys_info(left_inspector, right_inspector, table_name):
+def _get_primary_keys_info(
+    left_inspector, right_inspector, table_name, ignores
+):
     left_pk_list = _get_primary_keys(left_inspector, table_name)
     right_pk_list = _get_primary_keys(right_inspector, table_name)
 
+    left_pk_list = _discard_ignores(left_pk_list, ignores)
+    right_pk_list = _discard_ignores(right_pk_list, ignores)
+
     # process into dict
     left_pk = dict((elem, elem) for elem in left_pk_list)
     right_pk = dict((elem, elem) for elem in right_pk_list)
@@ -255,10 +295,13 @@ def _get_primary_keys(inspector, table_name):
     return inspector.get_primary_keys(table_name)
 
 
-def _get_indexes_info(left_inspector, right_inspector, table_name):
+def _get_indexes_info(left_inspector, right_inspector, table_name, ignores):
     left_index_list = _get_indexes(left_inspector, table_name)
     right_index_list = _get_indexes(right_inspector, table_name)
 
+    left_index_list = _discard_ignores_by_name(left_index_list, ignores)
+    right_index_list = _discard_ignores_by_name(right_index_list, ignores)
+
     # process into dict
     left_index = dict((elem['name'], elem) for elem in left_index_list)
     right_index = dict((elem['name'], elem) for elem in right_index_list)
@@ -270,10 +313,13 @@ def _get_indexes(inspector, table_name):
     return inspector.get_indexes(table_name)
 
 
-def _get_columns_info(left_inspector, right_inspector, table_name):
+def _get_columns_info(left_inspector, right_inspector, table_name, ignores):
     left_columns_list = _get_columns(left_inspector, table_name)
     right_columns_list = _get_columns(right_inspector, table_name)
 
+    left_columns_list = _discard_ignores_by_name(left_columns_list, ignores)
+    right_columns_list = _discard_ignores_by_name(right_columns_list, ignores)
+
     # process into dict
     left_columns = dict((elem['name'], elem) for elem in left_columns_list)
     right_columns = dict((elem['name'], elem) for elem in right_columns_list)
@@ -289,6 +335,14 @@ def _get_columns(inspector, table_name):
     return inspector.get_columns(table_name)
 
 
+def _discard_ignores_by_name(items, ignores):
+    return [item for item in items if item['name'] not in ignores]
+
+
+def _discard_ignores(items, ignores):
+    return [item for item in items if item not in ignores]
+
+
 def _process_types(column_dict):
     for column in column_dict:
         column_dict[column]['type'] = _process_type(

sqlalchemydiff/comparer.py

@@ -3,6 +3,7 @@
 from uuid import uuid4
 import json
 
+import six
 from sqlalchemy import inspect, create_engine
 from sqlalchemy_utils import create_database, drop_database, database_exists
 
@@ -103,3 +104,75 @@ def prepare_schema_from_models(uri, sqlalchemy_base):
     """Creates the database schema from the ``SQLAlchemy`` models. """
     engine = create_engine(uri)
     sqlalchemy_base.metadata.create_all(engine)
+
+
+class IgnoreManager:
+
+    allowed_identifiers = ['pk', 'fk', 'idx', 'col']
+
+    def __init__(self, ignores, separator=None):
+        self.separator = separator or '.'
+        self.parse(ignores or [])
+
+    def parse(self, ignores):
+        ignore, tables = {}, set()
+
+        for data in ignores:
+            self.validate_type(data)
+
+            if self.is_table_name(data):
+                tables.add(data.strip())
+            else:
+                self.validate_clause(data)
+                table_name, identifier, name = self.fetch_data_items(data)
+                self.validate_items(table_name, identifier, name)
+
+                ignore.setdefault(
+                    table_name, {}
+                ).setdefault(identifier, []).append(name)
+
+        self.__ignore = ignore
+        self.__tables = tables
+
+    def is_table_name(self, data):
+        return data.count(self.separator) == 0
+
+    def validate_type(self, data):
+        if not isinstance(data, six.string_types):
+            raise TypeError('{} is not a string'.format(data))
+
+    def validate_clause(self, data):
+        if len(data.split(self.separator)) != 3:
+            raise ValueError(
+                '{} is not a well formed clause: table_name.identifier.name'
+                .format(data)
+            )
+
+    def fetch_data_items(self, data):
+        return [item.strip() for item in data.split(self.separator)]
+
+    def validate_items(self, table_name, identifier, name):
+        if identifier not in self.allowed_identifiers:
+            raise ValueError(
+                '{} is invalid. It must be in {}'.format(
+                    identifier, self.allowed_identifiers
+                )
+            )
+
+        items = (table_name, identifier, name)
+        if not all(items):
+            raise ValueError(
+                '{} is not a well formed clause: table_name.identifier.name'
+                .format(self.separator.join(items))
+            )
+
+    def get(self, table_name, identifier):
+        return self.__ignore.get(table_name, {}).get(identifier, [])
+
+    @property
+    def ignore_tables(self):
+        return self.__tables.copy()
+
+    @property
+    def ignore_data(self):
+        return self.__ignore.copy()

sqlalchemydiff/util.py

@@ -0,0 +1,22 @@
+import os
+
+import pytest
+import yaml
+
+
+@pytest.fixture(scope="session")
+def project_root():
+    return os.path.dirname(os.path.dirname(__file__))
+
+
+@pytest.fixture(scope="session")
+def test_config(project_root):
+    config_file = os.path.join(project_root, "config", "config.yaml")
+    with open(config_file) as stream:
+        config = yaml.load(stream.read())
+    return config
+
+
+@pytest.fixture(scope="session")
+def db_uri(test_config):
+    return test_config['DB_URIS']['test']