Changeset - 12d4fc714b5a
[Not reviewed]
0 2 0
Lance Edgar - 8 years ago 2016-05-14 01:02:47
ledgar@sacfoodcoop.com
Give teeth to `ImportHandler.process_changes()` in new framework

I.e. make it send the diff emails by default.
2 files changed with 78 insertions and 20 deletions:
0 comments (0 inline, 0 general)
rattail/importing/handlers.py
Show inline comments
 
@@ -5,53 +5,60 @@
 
#  Copyright © 2010-2016 Lance Edgar
 
#
 
#  This file is part of Rattail.
 
#
 
#  Rattail is free software: you can redistribute it and/or modify it under the
 
#  terms of the GNU Affero General Public License as published by the Free
 
#  Software Foundation, either version 3 of the License, or (at your option)
 
#  any later version.
 
#
 
#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 
#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 
#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
 
#  more details.
 
#
 
#  You should have received a copy of the GNU Affero General Public License
 
#  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.
 
#
 
################################################################################
 
"""
 
Import Handlers
 
"""
 

	
 
from __future__ import unicode_literals, absolute_import
 

	
 
import sys
 
import datetime
 
import logging
 

	
 
import humanize
 

	
 
from rattail.time import make_utc
 
from rattail.util import OrderedDict
 
from rattail.mail import send_email
 

	
 
# TODO
 
from rattail.db.newimporting.handlers import RecordRenderer
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class ImportHandler(object):
 
    """
 
    Base class for all import handlers.
 
    """
 
    host_title = None
 
    local_title = None
 
    progress = None
 
    dry_run = False
 

	
 
    def __init__(self, config=None, **kwargs):
 
        self.config = config
 
        self.importers = self.get_importers()
 
        for key, value in kwargs.iteritems():
 
            setattr(self, key, value)
 

	
 
    def get_importers(self):
 
        """
 
        Returns a dict of all available importers, where the keys are model
 
        names and the values are importer factories.  All subclasses will want
 
@@ -159,48 +166,80 @@ class ImportHandler(object):
 
        self.rollback_local_transaction()
 

	
 
    def rollback_host_transaction(self):
 
        pass
 

	
 
    def rollback_local_transaction(self):
 
        pass
 

	
 
    def commit_transaction(self):
 
        self.commit_host_transaction()
 
        self.commit_local_transaction()
 

	
 
    def commit_host_transaction(self):
 
        pass
 

	
 
    def commit_local_transaction(self):
 
        pass
 

	
 
    def process_changes(self, changes):
 
        """
 
        This method is called any time changes occur, regardless of whether the
 
        import is running in "warnings" mode.  Default implementation does
 
        nothing; override as needed.
 
        """
 
        # TODO: This whole thing needs a re-write...but for now, waiting until
 
        # the old importer has really gone away, so we can share its email
 
        # template instead of bothering with something more complicated.
 

	
 
        if not self.warnings:
 
            return
 

	
 
        now = make_utc(datetime.datetime.utcnow(), tzinfo=True)
 
        data = {
 
            'local_title': self.local_title,
 
            'host_title': self.host_title,
 
            'argv': sys.argv,
 
            'runtime': humanize.naturaldelta(now - self.import_began),
 
            'changes': changes,
 
            'dry_run': self.dry_run,
 
            'render_record': RecordRenderer(self.config),
 
            'max_display': 15,
 
        }
 

	
 
        command = getattr(self, 'command', None)
 
        if command:
 
            data['command'] = '{} {}'.format(command.parent.name, command.name)
 
        else:
 
            data['command'] = None
 

	
 
        if command:
 
            key = '{}_{}_updates'.format(command.parent.name, command.name)
 
            key = key.replace('-', '_')
 
        else:
 
            key = 'rattail_import_updates'
 

	
 
        send_email(self.config, key, fallback_key='rattail_import_updates', data=data)
 

	
 

	
 
class FromSQLAlchemyHandler(ImportHandler):
 
    """
 
    Handler for imports for which the host data source is represented by a
 
    SQLAlchemy engine and ORM.
 
    """
 
    host_session = None
 

	
 
    def make_host_session(self):
 
        """
 
        Subclasses must override this to define the host database connection.
 
        """
 
        raise NotImplementedError
 

	
 
    def get_importer_kwargs(self, key, **kwargs):
 
        kwargs = super(FromSQLAlchemyHandler, self).get_importer_kwargs(key, **kwargs)
 
        kwargs.setdefault('host_session', self.host_session)
 
        return kwargs
 

	
 
    def begin_host_transaction(self):
 
        self.host_session = self.make_host_session()
 

	
 
    def rollback_host_transaction(self):
rattail/tests/importing/test_handlers.py
Show inline comments
 
@@ -136,50 +136,51 @@ class TestImportHandlerBasics(unittest.TestCase):
 
######################################################################
 
# fake import handler, tested mostly for basic coverage
 
######################################################################
 

	
 
class MockImportHandler(handlers.ImportHandler):
 

	
 
    def get_importers(self):
 
        return {'Product': MockImporter}
 

	
 
    def import_data(self, *keys, **kwargs):
 
        result = super(MockImportHandler, self).import_data(*keys, **kwargs)
 
        self._result = result
 
        return result
 

	
 

	
 
class TestImportHandlerImportData(ImporterTester, unittest.TestCase):
 

	
 
    sample_data = {
 
        '16oz': {'upc': '00074305001161', 'description': "Apple Cider Vinegar 16oz"},
 
        '32oz': {'upc': '00074305001321', 'description': "Apple Cider Vinegar 32oz"},
 
        '1gal': {'upc': '00074305011283', 'description': "Apple Cider Vinegar 1gal"},
 
    }
 

	
 
    def setUp(self):
 
        self.handler = MockImportHandler()
 
        self.importer = MockImporter()
 
        self.config = RattailConfig()
 
        self.handler = MockImportHandler(config=self.config)
 
        self.importer = MockImporter(config=self.config)
 

	
 
    def import_data(self, **kwargs):
 
        # must modify our importer in-place since we need the handler to return
 
        # that specific instance, below (because the host/local data context
 
        # managers reference that instance directly)
 
        self.importer._setup(**kwargs)
 
        with patch.object(self.handler, 'get_importer', Mock(return_value=self.importer)):
 
            result = self.handler.import_data('Product', **kwargs)
 
        if result:
 
            self.result = result['Product']
 
        else:
 
            self.result = [], [], []
 

	
 
    def test_invalid_importer_key_is_ignored(self):
 
        handler = handlers.ImportHandler()
 
        self.assertNotIn('InvalidKey', handler.importers)
 
        self.assertEqual(handler.import_data('InvalidKey'), {})
 

	
 
    def test_create(self):
 
        local = self.copy_data()
 
        del local['32oz']
 
        with self.host_data(self.sample_data):
 
            with self.local_data(local):
 
                self.import_data()
 
@@ -277,48 +278,74 @@ class TestImportHandlerImportData(ImporterTester, unittest.TestCase):
 
        local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}
 
        with self.host_data(self.sample_data):
 
            with self.local_data(local):
 
                self.import_data(max_total=1)
 
        self.assert_import_created()
 
        self.assert_import_updated()
 
        self.assert_import_deleted('bogus1')
 

	
 
    def test_dry_run(self):
 
        local = self.copy_data()
 
        del local['32oz']
 
        local['16oz']['description'] = "wrong description"
 
        local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}
 
        with self.host_data(self.sample_data):
 
            with self.local_data(local):
 
                self.import_data(dry_run=True)
 
        # TODO: maybe need a way to confirm no changes actually made due to dry
 
        # run; currently results still reflect "proposed" changes.  this rather
 
        # bogus test is here just for coverage sake
 
        self.assert_import_created('32oz')
 
        self.assert_import_updated('16oz')
 
        self.assert_import_deleted('bogus')
 

	
 
    def test_warnings_run(self):
 
        local = self.copy_data()
 
        del local['32oz']
 
        local['16oz']['description'] = "wrong description"
 
        local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}
 
        with self.host_data(self.sample_data):
 
            with self.local_data(local):
 
                with patch('rattail.importing.handlers.send_email') as send_email:
 
                    self.assertEqual(send_email.call_count, 0)
 
                    self.import_data(warnings=True, dry_run=True)
 
                    self.assertEqual(send_email.call_count, 1)
 
        # second time is just for more coverage...
 
        with self.host_data(self.sample_data):
 
            with self.local_data(local):
 
                with patch('rattail.importing.handlers.send_email') as send_email:
 
                    self.handler.command = Mock()
 
                    self.assertEqual(send_email.call_count, 0)
 
                    self.import_data(warnings=True)
 
                    self.assertEqual(send_email.call_count, 1)
 
        # TODO: maybe need a way to confirm no changes actually made due to dry
 
        # run; currently results still reflect "proposed" changes.  this rather
 
        # bogus test is here just for coverage sake
 
        self.assert_import_created('32oz')
 
        self.assert_import_updated('16oz')
 
        self.assert_import_deleted('bogus')
 

	
 

	
 
Session = orm.sessionmaker()
 

	
 

	
 
class MockFromSQLAlchemyHandler(handlers.FromSQLAlchemyHandler):
 

	
 
    def make_host_session(self):
 
        return Session()
 

	
 

	
 
class MockToSQLAlchemyHandler(handlers.ToSQLAlchemyHandler):
 

	
 
    def make_session(self):
 
        return Session()
 

	
 

	
 
class TestFromSQLAlchemyHandler(unittest.TestCase):
 

	
 
    def test_init(self):
 
        handler = handlers.FromSQLAlchemyHandler()
 
        self.assertRaises(NotImplementedError, handler.make_host_session)
 

	
 
    def test_get_importer_kwargs(self):
 
        session = object()
 
@@ -388,78 +415,70 @@ class TestToSQLAlchemyHandler(unittest.TestCase):
 
        handler = handlers.ToSQLAlchemyHandler(session=session)
 
        self.assertIs(handler.session, session)
 
        with patch.object(handler, 'session') as session:
 
            handler.rollback_local_transaction()
 
            session.rollback.assert_called_once_with()
 
            self.assertFalse(session.commit.called)
 
        # self.assertIsNone(handler.session)
 

	
 

	
 
######################################################################
 
# fake bulk import handler, tested mostly for basic coverage
 
######################################################################
 

	
 
class MockBulkImportHandler(handlers.BulkToPostgreSQLHandler):
 

	
 
    def get_importers(self):
 
        return {'Department': MockBulkImporter}
 

	
 
    def make_session(self):
 
        return Session()
 

	
 

	
 
class TestBulkImportHandler(RattailTestCase, ImporterTester):
 

	
 
    importer_class = MockBulkImporter
 

	
 
    sample_data = {
 
        'grocery': {'number': 1, 'name': "Grocery", 'uuid': 'decd909a194011e688093ca9f40bc550'},
 
        'bulk': {'number': 2, 'name': "Bulk", 'uuid': 'e633d54c194011e687e33ca9f40bc550'},
 
        'hba': {'number': 3, 'name': "HBA", 'uuid': 'e2bad79e194011e6a4783ca9f40bc550'},
 
    }
 

	
 
    def setUp(self):
 
        self.setup_rattail()
 
        self.tempio = TempIO()
 
        self.config.set('rattail', 'workdir', self.tempio.realpath())
 
        self.handler = MockBulkImportHandler(config=self.config)
 
        self.importer = MockBulkImporter(config=self.config)
 

	
 
    def tearDown(self):
 
        self.teardown_rattail()
 
        self.tempio = None
 

	
 
    def postgresql(self):
 
        return self.config.rattail_engine.url.get_dialect().name == 'postgresql'
 

	
 
    def import_data(self, **kwargs):
 
        # must modify our importer in-place since we need the handler to return
 
        # that specific instance, below (because the host/local data context
 
        # managers reference that instance directly)
 
        self.importer._setup(**kwargs)
 
        self.importer.session = self.session
 
        with patch.object(self.handler, 'get_importer', Mock(return_value=self.importer)):
 
            result = self.handler.import_data('Department', **kwargs)
 
    def import_data(self, host_data=None, **kwargs):
 
        if host_data is None:
 
            host_data = list(self.copy_data().itervalues())
 
        with patch.object(self.importer_class, 'normalize_host_data', Mock(return_value=host_data)):
 
            with patch.object(self.handler, 'make_session', Mock(return_value=self.session)):
 
                return self.handler.import_data('Department', **kwargs)
 

	
 
    def test_invalid_importer_key_is_ignored(self):
 
        handler = MockBulkImportHandler()
 
        self.assertNotIn('InvalidKey', handler.importers)
 
        self.assertEqual(handler.import_data('InvalidKey'), {})
 

	
 
    def assert_import_created(self, *keys):
 
        pass
 

	
 
    def assert_import_updated(self, *keys):
 
        pass
 

	
 
    def assert_import_deleted(self, *keys):
 
        pass
 

	
 
    def test_normal_run(self):
 
        if self.postgresql():
 
            with self.host_data(self.sample_data):
 
                with self.local_data({}):
 
                    self.import_data()
 
            self.import_data()
 

	
 
    def test_dry_run(self):
 
        if self.postgresql():
 
            with self.host_data(self.sample_data):
 
                with self.local_data({}):
 
                    self.import_data(dry_run=True)
 
            self.import_data(dry_run=True)
0 comments (0 inline, 0 general)