rattail-project/rattail Changeset

Changeset - 10040a8c3bf4

Parent rev.

Child rev.

[Not reviewed]

0 7 1

Lance Edgar - 8 years ago 2016-05-16 19:41:21
ledgar@sacfoodcoop.com

More tweaks to new importer framework

* Add `ImportSubcommand.handler_spec` for simpler subclass config
* Pass `args` all the way from command -> handler -> importer
* Add `FromRattailHandler` for convenience

8 files changed with 144 insertions and 125 deletions:

rattail/commands/importing.py

rattail/importing/__init__.py

rattail/importing/rattail.py

rattail/tests/commands/test_importing.py

rattail/tests/importing/__init__.py

rattail/tests/importing/lib.py

100

rattail/tests/importing/test_importers.py

rattail/tests/importing/test_rattail.py

0 comments (0 inline, 0 general)

rattail/commands/importing.py

➞

Show inline comments

@@ @@ -30,37 +30,40 @@ import logging @@
 from rattail.commands.core import Subcommand, date_argument
 from rattail.util import load_object
 log = logging.getLogger(__name__)
 class ImportSubcommand(Subcommand):
     """
     Base class for subcommands which use the (new) data importing system.
     """
     handler_spec = None
     # TODO: move this into Subcommand or something..
     parent_name = None
     def __init__(self, *args, **kwargs):
         super(ImportSubcommand, self).__init__(*args, **kwargs)
         if self.parent:
             self.parent_name = self.parent.name
     def get_handler_factory(self):
         """
         Subclasses must override this, and return a callable that creates an
         import handler instance which the command should use.
         """
         if self.handler_spec:
             return load_object(self.handler_spec)
         raise NotImplementedError
     def get_handler(self, **kwargs):
         """
         Returns a handler instance to be used by the command.
         """
         factory = self.get_handler_factory()
         kwargs.setdefault('config', getattr(self, 'config', None))
         kwargs.setdefault('command', self)
         kwargs.setdefault('progress', self.progress)
         if 'args' in kwargs:
             args = kwargs['args']
@@ @@ -160,24 +163,25 @@ class ImportSubcommand(Subcommand): @@
         kwargs = {
             'dry_run': args.dry_run,
             'warnings': args.warnings,
             'create': args.create,
             'max_create': args.max_create,
             'update': args.update,
             'max_update': args.max_update,
             'delete': args.delete,
             'max_delete': args.max_delete,
             'max_total': args.max_total,
             'progress': self.progress,
             'args': args,
+        }
         handler.import_data(*models, **kwargs)
         # TODO: should this logging happen elsewhere / be customizable?
         if args.dry_run:
             log.info("dry run, so transaction was rolled back")
         else:
             log.info("transaction was committed")
 class ImportRattail(ImportSubcommand):
     """

rattail/importing/__init__.py

➞

Show inline comments

@@ @@ -21,14 +21,14 @@ @@
+#
 ################################################################################
 """
 Data Importing Framework
 """
 from __future__ import unicode_literals, absolute_import
 from .importers import Importer, FromQuery
 from .sqlalchemy import FromSQLAlchemy, ToSQLAlchemy
 from .postgresql import BulkToPostgreSQL
 from .handlers import ImportHandler, FromSQLAlchemyHandler, ToSQLAlchemyHandler, BulkToPostgreSQLHandler
 from .rattail import ToRattailHandler
+from .rattail import FromRattailHandler, ToRattailHandler
 from . import model

rattail/importing/rattail.py

➞

Show inline comments

@@ @@ -24,35 +24,45 @@ @@
 Rattail -> Rattail data import
 """
 from __future__ import unicode_literals, absolute_import
 from rattail.db import Session
 from rattail.importing import model
 from rattail.importing.handlers import FromSQLAlchemyHandler, ToSQLAlchemyHandler
 from rattail.importing.sqlalchemy import FromSQLAlchemy
 from rattail.util import OrderedDict
 class FromRattailHandler(FromSQLAlchemyHandler):
     """
     Base class for import handlers which target a Rattail database on the local side.
     """
     host_title = "Rattail"
     def make_host_session(self):
         return Session()
 class ToRattailHandler(ToSQLAlchemyHandler):
     """
     Base class for import handlers which target a Rattail database on the local side.
     """
     local_title = "Rattail"
     def make_session(self):
         return Session()
-class FromRattailToRattail(FromSQLAlchemyHandler, ToRattailHandler):
+class FromRattailToRattail(FromRattailHandler, ToRattailHandler):
     """
     Handler for Rattail -> Rattail data import.
     """
     local_title = "Rattail (local)"
     dbkey = 'host'
     @property
     def host_title(self):
         return "Rattail ({})".format(self.dbkey)
     def make_host_session(self):
         return Session(bind=self.config.rattail_engines[self.dbkey])

rattail/tests/commands/test_importing.py

➞

Show inline comments

@@ @@ -26,24 +26,27 @@ class MockImport(importing.ImportSubcommand): @@
 class TestImportSubcommandBasics(TestCase):
     # TODO: lame, here only for coverage
     def test_parent_name(self):
         parent = Object(name='milo')
         command = importing.ImportSubcommand(parent=parent)
         self.assertEqual(command.parent_name, 'milo')
     def test_get_handler_factory(self):
         command = importing.ImportSubcommand()
         self.assertRaises(NotImplementedError, command.get_handler_factory)
         command.handler_spec = 'rattail.importing.rattail:FromRattailToRattail'
         factory = command.get_handler_factory()
         self.assertIs(factory, FromRattailToRattail)
     def test_get_handler(self):
         # no config
         command = MockImport()
         handler = command.get_handler()
         self.assertIs(type(handler), MockImportHandler)
         self.assertIsNone(handler.config)
         # with config
         config = RattailConfig()
         command = MockImport(config=config)

rattail/tests/importing/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 import copy
 from contextlib import contextmanager
 from mock import patch
 from rattail.tests import NullProgress
 class ImporterTester(object):
     """
     Mixin for importer test suites.
     """
     importer_class = None
     sample_data = {}
     def make_importer(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         kwargs.setdefault('progress', NullProgress)
         return self.importer_class(**kwargs)
     def copy_data(self):
         return copy.deepcopy(self.sample_data)
     @contextmanager
     def host_data(self, data):
         self._host_data = data
         host_data = [self.importer.normalize_host_object(obj) for obj in data.itervalues()]
         with patch.object(self.importer, 'normalize_host_data') as normalize:
             normalize.return_value = host_data
             yield
     @contextmanager
     def local_data(self, data):
         self._local_data = data
         local_data = {}
         for key, obj in data.iteritems():
             normal = self.importer.normalize_local_object(obj)
             local_data[self.importer.get_key(normal)] = {'object': obj, 'data': normal}
         with patch.object(self.importer, 'cache_local_data') as cache:
             cache.return_value = local_data
             yield
     def import_data(self, **kwargs):
         self.result = self.importer.import_data(**kwargs)
     def assert_import_created(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(created), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, host_data in created:
                 if self.importer.get_key(host_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not created when importing with {}".format(key, self.importer))
     def assert_import_updated(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(updated), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, local_data, host_data in updated:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not updated when importing with {}".format(key, self.importer))
     def assert_import_deleted(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(deleted), len(keys))
         for key in keys:
             key = self.importer.get_key(self._local_data[key])
             found = False
             for local_object, local_data in deleted:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not deleted when importing with {}".format(key, self.importer))
 from .lib import ImporterTester

rattail/tests/importing/lib.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 import copy
 from contextlib import contextmanager
 from mock import patch
 from rattail.tests import NullProgress
 class ImporterTester(object):
     """
     Mixin for importer test suites.
     """
     handler_class = None
     importer_class = None
     sample_data = {}
     def make_handler(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         return self.handler_class(**kwargs)
     def make_importer(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         kwargs.setdefault('progress', NullProgress)
         return self.importer_class(**kwargs)
     def copy_data(self):
         return copy.deepcopy(self.sample_data)
     @contextmanager
     def host_data(self, data):
         self._host_data = data
         host_data = [self.importer.normalize_host_object(obj) for obj in data.itervalues()]
         with patch.object(self.importer, 'normalize_host_data') as normalize:
             normalize.return_value = host_data
             yield
     @contextmanager
     def local_data(self, data):
         self._local_data = data
         local_data = {}
         for key, obj in data.iteritems():
             normal = self.importer.normalize_local_object(obj)
             local_data[self.importer.get_key(normal)] = {'object': obj, 'data': normal}
         with patch.object(self.importer, 'cache_local_data') as cache:
             cache.return_value = local_data
             yield
     def import_data(self, host_data=None, local_data=None, **kwargs):
         if host_data is None:
             host_data = self.sample_data
         if local_data is None:
             local_data = self.sample_data
         with self.host_data(host_data):
             with self.local_data(local_data):
                 self.result = self.importer.import_data(**kwargs)
     def assert_import_created(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(created), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, host_data in created:
                 if self.importer.get_key(host_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not created when importing with {}".format(key, self.importer))
     def assert_import_updated(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(updated), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, local_data, host_data in updated:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not updated when importing with {}".format(key, self.importer))
     def assert_import_deleted(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(deleted), len(keys))
         for key in keys:
             key = self.importer.get_key(self._local_data[key])
             found = False
             for local_object, local_data in deleted:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not deleted when importing with {}".format(key, self.importer))

rattail/tests/importing/test_importers.py

➞

Show inline comments

@@ @@ -195,136 +195,112 @@ class TestMockImporter(ImporterTester, TestCase): @@
     sample_data = {
         '16oz': {'upc': '00074305001161', 'description': "Apple Cider Vinegar 16oz"},
         '32oz': {'upc': '00074305001321', 'description': "Apple Cider Vinegar 32oz"},
         '1gal': {'upc': '00074305011283', 'description': "Apple Cider Vinegar 1gal"},
+    }
     def setUp(self):
         self.importer = self.make_importer()
     def test_create(self):
         local = self.copy_data()
         del local['32oz']
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data()
         self.import_data(local_data=local)
         self.assert_import_created('32oz')
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_create_empty(self):
         with self.host_data({}):
             with self.local_data({}):
                 self.import_data()
         self.import_data(host_data={}, local_data={})
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_update(self):
         local = self.copy_data()
         local['16oz']['description'] = "wrong description"
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data()
         self.import_data(local_data=local)
         self.assert_import_created()
         self.assert_import_updated('16oz')
         self.assert_import_deleted()
     def test_delete(self):
         local = self.copy_data()
         local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True)
         self.import_data(local_data=local, delete=True)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted('bogus')
     def test_duplicate(self):
         host = self.copy_data()
         host['32oz-dupe'] = host['32oz']
         with self.host_data(host):
             with self.local_data(self.sample_data):
                 self.import_data()
         self.import_data(host_data=host)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_max_create(self):
         local = self.copy_data()
         del local['16oz']
         del local['1gal']
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_create=1)
         self.import_data(local_data=local, max_create=1)
         self.assert_import_created('16oz')
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_max_total_create(self):
         local = self.copy_data()
         del local['16oz']
         del local['1gal']
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_total=1)
         self.import_data(local_data=local, max_total=1)
         self.assert_import_created('16oz')
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_max_update(self):
         local = self.copy_data()
         local['16oz']['description'] = "wrong"
         local['1gal']['description'] = "wrong"
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_update=1)
         self.import_data(local_data=local, max_update=1)
         self.assert_import_created()
         self.assert_import_updated('16oz')
         self.assert_import_deleted()
     def test_max_total_update(self):
         local = self.copy_data()
         local['16oz']['description'] = "wrong"
         local['1gal']['description'] = "wrong"
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_total=1)
         self.import_data(local_data=local, max_total=1)
         self.assert_import_created()
         self.assert_import_updated('16oz')
         self.assert_import_deleted()
     def test_max_delete(self):
         local = self.copy_data()
         local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}
         local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True, max_delete=1)
         self.import_data(local_data=local, delete=True, max_delete=1)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted('bogus1')
     def test_max_total_delete(self):
         local = self.copy_data()
         local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}
         local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True, max_total=1)
         self.import_data(local_data=local, delete=True, max_total=1)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted('bogus1')
     def test_max_total_delete_with_changes(self):
         local = self.copy_data()
         del local['16oz']
         local['32oz']['description'] = "wrong"
         local['1gal']['description'] = "wrong"
         local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}
         local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True, max_total=3)
         self.import_data(local_data=local, delete=True, max_total=3)
         self.assert_import_created('16oz')
         self.assert_import_updated('32oz', '1gal')
         self.assert_import_deleted()

rattail/tests/importing/test_rattail.py

➞

Show inline comments

@@ @@ -2,24 +2,25 @@ @@
 from __future__ import unicode_literals, absolute_import
 from unittest import TestCase
 import sqlalchemy as sa
 from mock import patch
 from fixture import TempIO
 from rattail.db import model, Session, SessionBase, auth
 from rattail.importing import rattail as rattail_importing
 from rattail.tests import RattailMixin, RattailTestCase
 from rattail.tests.importing import ImporterTester
 class DualRattailMixin(RattailMixin):
     def setup_rattail(self):
         super(DualRattailMixin, self).setup_rattail()
         if 'host' not in self.config.rattail_engines:
             self.config.rattail_engines['host'] = sa.create_engine('sqlite://')
         self.host_engine = self.config.rattail_engines['host']
         self.config.setdefault('rattail.db', 'keys', 'default, host')
@@ @@ -34,28 +35,36 @@ class DualRattailMixin(RattailMixin): @@
         self.host_session.close()
         model = self.get_rattail_model()
         model.Base.metadata.drop_all(bind=self.config.rattail_engines['host'])
         if hasattr(self, 'tempio'):
             self.tempio = None
 class DualRattailTestCase(DualRattailMixin, TestCase):
     pass
 class TestFromRattailToRattail(DualRattailTestCase):
 class TestFromRattailHandler(RattailTestCase, ImporterTester):
     handler_class = rattail_importing.FromRattailHandler
     def make_handler(self, **kwargs):
         return rattail_importing.FromRattailToRattail(self.config, **kwargs)
     def test_make_host_session(self):
         handler = self.make_handler()
         session = handler.make_host_session()
         self.assertIsInstance(session, SessionBase)
         self.assertIs(session.bind, self.config.rattail_engine)
 class TestFromRattailToRattail(DualRattailTestCase, ImporterTester):
     handler_class = rattail_importing.FromRattailToRattail
     def test_host_title(self):
         handler = self.make_handler()
         self.assertEqual(handler.host_title, "Rattail (host)")
     # TODO
     def test_default_keys(self):
         handler = self.make_handler()
         handler.get_default_keys()
     def test_make_session(self):
         handler = self.make_handler()

0 comments (0 inline, 0 general)