rattail-project/rattail Changeset

Changeset - 10040a8c3bf4

Parent rev.

Child rev.

[Not reviewed]

0 7 1

Lance Edgar - 8 years ago 2016-05-16 19:41:21
ledgar@sacfoodcoop.com

More tweaks to new importer framework

* Add `ImportSubcommand.handler_spec` for simpler subclass config
* Pass `args` all the way from command -> handler -> importer
* Add `FromRattailHandler` for convenience

8 files changed with 144 insertions and 125 deletions:

rattail/commands/importing.py

rattail/importing/__init__.py

rattail/importing/rattail.py

rattail/tests/commands/test_importing.py

rattail/tests/importing/__init__.py

rattail/tests/importing/lib.py

100

rattail/tests/importing/test_importers.py

rattail/tests/importing/test_rattail.py

0 comments (0 inline, 0 general)

rattail/commands/importing.py

➞

Show inline comments

@@ @@ -36,12 +36,13 @@ log = logging.getLogger(__name__) @@
 class ImportSubcommand(Subcommand):
     """
     Base class for subcommands which use the (new) data importing system.
     """
     handler_spec = None
     # TODO: move this into Subcommand or something..
     parent_name = None
     def __init__(self, *args, **kwargs):
         super(ImportSubcommand, self).__init__(*args, **kwargs)
         if self.parent:
@@ @@ -49,12 +50,14 @@ class ImportSubcommand(Subcommand): @@
     def get_handler_factory(self):
         """
         Subclasses must override this, and return a callable that creates an
         import handler instance which the command should use.
         """
         if self.handler_spec:
             return load_object(self.handler_spec)
         raise NotImplementedError
     def get_handler(self, **kwargs):
         """
         Returns a handler instance to be used by the command.
         """
@@ @@ -166,12 +169,13 @@ class ImportSubcommand(Subcommand): @@
             'update': args.update,
             'max_update': args.max_update,
             'delete': args.delete,
             'max_delete': args.max_delete,
             'max_total': args.max_total,
             'progress': self.progress,
             'args': args,
+        }
         handler.import_data(*models, **kwargs)
         # TODO: should this logging happen elsewhere / be customizable?
         if args.dry_run:
             log.info("dry run, so transaction was rolled back")

rattail/importing/__init__.py

➞

Show inline comments

@@ @@ -27,8 +27,8 @@ Data Importing Framework @@
 from __future__ import unicode_literals, absolute_import
 from .importers import Importer, FromQuery
 from .sqlalchemy import FromSQLAlchemy, ToSQLAlchemy
 from .postgresql import BulkToPostgreSQL
 from .handlers import ImportHandler, FromSQLAlchemyHandler, ToSQLAlchemyHandler, BulkToPostgreSQLHandler
 from .rattail import ToRattailHandler
+from .rattail import FromRattailHandler, ToRattailHandler
 from . import model

rattail/importing/rattail.py

➞

Show inline comments

@@ @@ -30,23 +30,33 @@ from rattail.db import Session @@
 from rattail.importing import model
 from rattail.importing.handlers import FromSQLAlchemyHandler, ToSQLAlchemyHandler
 from rattail.importing.sqlalchemy import FromSQLAlchemy
 from rattail.util import OrderedDict
 class FromRattailHandler(FromSQLAlchemyHandler):
     """
     Base class for import handlers which target a Rattail database on the local side.
     """
     host_title = "Rattail"
     def make_host_session(self):
         return Session()
 class ToRattailHandler(ToSQLAlchemyHandler):
     """
     Base class for import handlers which target a Rattail database on the local side.
     """
     local_title = "Rattail"
     def make_session(self):
         return Session()
-class FromRattailToRattail(FromSQLAlchemyHandler, ToRattailHandler):
+class FromRattailToRattail(FromRattailHandler, ToRattailHandler):
     """
     Handler for Rattail -> Rattail data import.
     """
     local_title = "Rattail (local)"
     dbkey = 'host'

rattail/tests/commands/test_importing.py

➞

Show inline comments

@@ @@ -32,12 +32,15 @@ class TestImportSubcommandBasics(TestCase): @@
         command = importing.ImportSubcommand(parent=parent)
         self.assertEqual(command.parent_name, 'milo')
     def test_get_handler_factory(self):
         command = importing.ImportSubcommand()
         self.assertRaises(NotImplementedError, command.get_handler_factory)
         command.handler_spec = 'rattail.importing.rattail:FromRattailToRattail'
         factory = command.get_handler_factory()
         self.assertIs(factory, FromRattailToRattail)
     def test_get_handler(self):
         # no config
         command = MockImport()
         handler = command.get_handler()

rattail/tests/importing/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 import copy
 from contextlib import contextmanager
 from mock import patch
 from rattail.tests import NullProgress
 class ImporterTester(object):
     """
     Mixin for importer test suites.
     """
     importer_class = None
     sample_data = {}
     def make_importer(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         kwargs.setdefault('progress', NullProgress)
         return self.importer_class(**kwargs)
     def copy_data(self):
         return copy.deepcopy(self.sample_data)
     @contextmanager
     def host_data(self, data):
         self._host_data = data
         host_data = [self.importer.normalize_host_object(obj) for obj in data.itervalues()]
         with patch.object(self.importer, 'normalize_host_data') as normalize:
             normalize.return_value = host_data
             yield
     @contextmanager
     def local_data(self, data):
         self._local_data = data
         local_data = {}
         for key, obj in data.iteritems():
             normal = self.importer.normalize_local_object(obj)
             local_data[self.importer.get_key(normal)] = {'object': obj, 'data': normal}
         with patch.object(self.importer, 'cache_local_data') as cache:
             cache.return_value = local_data
             yield
     def import_data(self, **kwargs):
         self.result = self.importer.import_data(**kwargs)
     def assert_import_created(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(created), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, host_data in created:
                 if self.importer.get_key(host_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not created when importing with {}".format(key, self.importer))
     def assert_import_updated(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(updated), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, local_data, host_data in updated:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not updated when importing with {}".format(key, self.importer))
     def assert_import_deleted(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(deleted), len(keys))
         for key in keys:
             key = self.importer.get_key(self._local_data[key])
             found = False
             for local_object, local_data in deleted:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not deleted when importing with {}".format(key, self.importer))
 from .lib import ImporterTester

rattail/tests/importing/lib.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 import copy
 from contextlib import contextmanager
 from mock import patch
 from rattail.tests import NullProgress
 class ImporterTester(object):
     """
     Mixin for importer test suites.
     """
     handler_class = None
     importer_class = None
     sample_data = {}
     def make_handler(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         return self.handler_class(**kwargs)
     def make_importer(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         kwargs.setdefault('progress', NullProgress)
         return self.importer_class(**kwargs)
     def copy_data(self):
         return copy.deepcopy(self.sample_data)
     @contextmanager
     def host_data(self, data):
         self._host_data = data
         host_data = [self.importer.normalize_host_object(obj) for obj in data.itervalues()]
         with patch.object(self.importer, 'normalize_host_data') as normalize:
             normalize.return_value = host_data
             yield
     @contextmanager
     def local_data(self, data):
         self._local_data = data
         local_data = {}
         for key, obj in data.iteritems():
             normal = self.importer.normalize_local_object(obj)
             local_data[self.importer.get_key(normal)] = {'object': obj, 'data': normal}
         with patch.object(self.importer, 'cache_local_data') as cache:
             cache.return_value = local_data
             yield
     def import_data(self, host_data=None, local_data=None, **kwargs):
         if host_data is None:
             host_data = self.sample_data
         if local_data is None:
             local_data = self.sample_data
         with self.host_data(host_data):
             with self.local_data(local_data):
                 self.result = self.importer.import_data(**kwargs)
     def assert_import_created(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(created), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, host_data in created:
                 if self.importer.get_key(host_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not created when importing with {}".format(key, self.importer))
     def assert_import_updated(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(updated), len(keys))
         for key in keys:
             key = self.importer.get_key(self._host_data[key])
             found = False
             for local_object, local_data, host_data in updated:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not updated when importing with {}".format(key, self.importer))
     def assert_import_deleted(self, *keys):
         created, updated, deleted = self.result
         self.assertEqual(len(deleted), len(keys))
         for key in keys:
             key = self.importer.get_key(self._local_data[key])
             found = False
             for local_object, local_data in deleted:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not deleted when importing with {}".format(key, self.importer))

rattail/tests/importing/test_importers.py

➞

Show inline comments

@@ @@ -201,130 +201,106 @@ class TestMockImporter(ImporterTester, TestCase): @@
     def setUp(self):
         self.importer = self.make_importer()
     def test_create(self):
         local = self.copy_data()
         del local['32oz']
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data()
         self.import_data(local_data=local)
         self.assert_import_created('32oz')
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_create_empty(self):
         with self.host_data({}):
             with self.local_data({}):
                 self.import_data()
         self.import_data(host_data={}, local_data={})
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_update(self):
         local = self.copy_data()
         local['16oz']['description'] = "wrong description"
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data()
         self.import_data(local_data=local)
         self.assert_import_created()
         self.assert_import_updated('16oz')
         self.assert_import_deleted()
     def test_delete(self):
         local = self.copy_data()
         local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True)
         self.import_data(local_data=local, delete=True)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted('bogus')
     def test_duplicate(self):
         host = self.copy_data()
         host['32oz-dupe'] = host['32oz']
         with self.host_data(host):
             with self.local_data(self.sample_data):
                 self.import_data()
         self.import_data(host_data=host)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_max_create(self):
         local = self.copy_data()
         del local['16oz']
         del local['1gal']
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_create=1)
         self.import_data(local_data=local, max_create=1)
         self.assert_import_created('16oz')
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_max_total_create(self):
         local = self.copy_data()
         del local['16oz']
         del local['1gal']
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_total=1)
         self.import_data(local_data=local, max_total=1)
         self.assert_import_created('16oz')
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_max_update(self):
         local = self.copy_data()
         local['16oz']['description'] = "wrong"
         local['1gal']['description'] = "wrong"
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_update=1)
         self.import_data(local_data=local, max_update=1)
         self.assert_import_created()
         self.assert_import_updated('16oz')
         self.assert_import_deleted()
     def test_max_total_update(self):
         local = self.copy_data()
         local['16oz']['description'] = "wrong"
         local['1gal']['description'] = "wrong"
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(max_total=1)
         self.import_data(local_data=local, max_total=1)
         self.assert_import_created()
         self.assert_import_updated('16oz')
         self.assert_import_deleted()
     def test_max_delete(self):
         local = self.copy_data()
         local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}
         local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True, max_delete=1)
         self.import_data(local_data=local, delete=True, max_delete=1)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted('bogus1')
     def test_max_total_delete(self):
         local = self.copy_data()
         local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}
         local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True, max_total=1)
         self.import_data(local_data=local, delete=True, max_total=1)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted('bogus1')
     def test_max_total_delete_with_changes(self):
         local = self.copy_data()
         del local['16oz']
         local['32oz']['description'] = "wrong"
         local['1gal']['description'] = "wrong"
         local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}
         local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data(delete=True, max_total=3)
         self.import_data(local_data=local, delete=True, max_total=3)
         self.assert_import_created('16oz')
         self.assert_import_updated('32oz', '1gal')
         self.assert_import_deleted()

rattail/tests/importing/test_rattail.py

➞

Show inline comments

@@ @@ -8,12 +8,13 @@ import sqlalchemy as sa @@
 from mock import patch
 from fixture import TempIO
 from rattail.db import model, Session, SessionBase, auth
 from rattail.importing import rattail as rattail_importing
 from rattail.tests import RattailMixin, RattailTestCase
 from rattail.tests.importing import ImporterTester
 class DualRattailMixin(RattailMixin):
     def setup_rattail(self):
         super(DualRattailMixin, self).setup_rattail()
@@ @@ -40,16 +41,24 @@ class DualRattailMixin(RattailMixin): @@
 class DualRattailTestCase(DualRattailMixin, TestCase):
     pass
 class TestFromRattailToRattail(DualRattailTestCase):
 class TestFromRattailHandler(RattailTestCase, ImporterTester):
     handler_class = rattail_importing.FromRattailHandler
     def make_handler(self, **kwargs):
         return rattail_importing.FromRattailToRattail(self.config, **kwargs)
     def test_make_host_session(self):
         handler = self.make_handler()
         session = handler.make_host_session()
         self.assertIsInstance(session, SessionBase)
         self.assertIs(session.bind, self.config.rattail_engine)
 class TestFromRattailToRattail(DualRattailTestCase, ImporterTester):
     handler_class = rattail_importing.FromRattailToRattail
     def test_host_title(self):
         handler = self.make_handler()
         self.assertEqual(handler.host_title, "Rattail (host)")
     # TODO

0 comments (0 inline, 0 general)