rattail-project/rattail Changeset

Changeset - 1704d9e0251e

Parent rev.

Child rev.

[Not reviewed]

0 19 5

Lance Edgar - 8 years ago 2016-05-14 00:26:19
ledgar@sacfoodcoop.com

Add new bulk PostgreSQL and Rattail->Rattail importers

Plus tests, sort of..plenty of stubs in here still.

24 files changed with 896 insertions and 56 deletions:

MANIFEST.in

covered.cfg

rattail/commands/importing.py

rattail/db/changes.py

rattail/db/sync/win32.py

rattail/importing/__init__.py

rattail/importing/handlers.py

rattail/importing/importers.py

rattail/importing/postgresql.py

131

rattail/importing/rattail.py

rattail/importing/rattail_bulk.py

213

rattail/importing/sqlalchemy.py

rattail/tests/__init__.py

rattail/tests/commands/test_core.py

rattail/tests/commands/test_importing.py

rattail/tests/importing/__init__.py

rattail/tests/importing/test_handlers.py

rattail/tests/importing/test_importers.py

rattail/tests/importing/test_postgresql.py

199

rattail/tests/importing/test_rattail.py

rattail/tests/importing/test_rattail_bulk.py

rattail/tests/importing/test_sqlalchemy.py

setup.cfg

setup.py

0 comments (0 inline, 0 general)

MANIFEST.in

➞

Show inline comments

 # -*- mode: conf -*-
 include *.cfg
 include *.txt
 include *.rst
 recursive-include rattail/data *
 include rattail/contrib/init.d/*

covered.cfg

➞

Show inline comments

@@ new file 100644 @@
 [nosetests]
 nocapture = 1
 tests = rattail.tests.test_barcodes,
         rattail.tests.commands.test_importing,
         rattail.tests.db.test_core,
         rattail.tests.db.model.test_core,
         rattail.tests.db.model.test_customers,
         rattail.tests.db.model.test_datasync,
         rattail.tests.db.model.test_org,
         rattail.tests.db.model.test_people,
         rattail.tests.filemon.test_actions,
         rattail.tests.filemon.test_config,
         rattail.tests.filemon.test_util,
         rattail.tests.importing
 with-coverage = 1
 cover-erase = 1
 cover-package = rattail.barcodes,
         rattail.commands.importing,
         rattail.db.core,
         rattail.db.model.core,
         rattail.db.model.customers,
         rattail.db.model.datasync,
         rattail.db.model.org,
         rattail.db.model.people,
         rattail.enum,
         rattail.filemon.actions,
         rattail.filemon.config,
         rattail.filemon.util,
         rattail.importing
 cover-inclusive = 1
 cover-min-percentage = 100
 cover-html-dir = htmlcov

rattail/commands/importing.py

➞

Show inline comments

@@ @@ -60,13 +60,18 @@ class ImportSubcommand(Subcommand): @@
         """
         factory = self.get_handler_factory()
         kwargs.setdefault('config', getattr(self, 'config', None))
         kwargs.setdefault('command', self)
         kwargs.setdefault('progress', self.progress)
         if 'args' in kwargs:
             kwargs.setdefault('dry_run', kwargs['args'].dry_run)
             args = kwargs['args']
             kwargs.setdefault('dry_run', args.dry_run)
             # kwargs.setdefault('max_create', args.max_create)
             # kwargs.setdefault('max_update', args.max_update)
             # kwargs.setdefault('max_delete', args.max_delete)
             # kwargs.setdefault('max_total', args.max_total)
         kwargs = self.get_handler_kwargs(**kwargs)
         return factory(**kwargs)
     def get_handler_kwargs(self, **kwargs):
         """
         Return a dict of kwargs to be passed to the handler factory.
@@ @@ -149,13 +154,23 @@ class ImportSubcommand(Subcommand): @@
         handler = self.get_handler(args=args)
         models = args.models or handler.get_default_keys()
         log.debug("using handler: {}".format(handler))
         log.debug("importing models: {}".format(models))
         log.debug("args are: {}".format(args))
         handler.import_data(*models)
         kwargs = {
             'dry_run': args.dry_run,
             'warnings': args.warnings,
             'max_create': args.max_create,
             'max_update': args.max_update,
             'max_delete': args.max_delete,
             'max_total': args.max_total,
             'progress': self.progress,
+        }
         handler.import_data(*models, **kwargs)
         # TODO: should this logging happen elsewhere / be customizable?
         if args.dry_run:
             log.info("dry run, so transaction was rolled back")
         else:
             log.info("transaction was committed")

rattail/db/changes.py

➞

Show inline comments

@@ @@ -99,17 +99,18 @@ class ChangeRecorder(object): @@
+    )
     def __call__(self, session, flush_context, instances):
         """
         Method invoked when session ``before_flush`` event occurs.
         """
         # TODO: Not sure if our event replaces the one registered by Continuum,
         # or what.  But this appears to be necessary to keep that system
         # working when we enable ours...
         if versioning_manager:
             versioning_manager.before_flush(session, flush_context, instances)
         # TODO: what a mess, need to look into this again at some point...
         # # TODO: Not sure if our event replaces the one registered by Continuum,
         # # or what.  But this appears to be necessary to keep that system
         # # working when we enable ours...
         # if versioning_manager:
         #     versioning_manager.before_flush(session, flush_context, instances)
         for obj in session.deleted:
             if not self.ignore_object(obj):
                 self.process_deleted_object(session, obj)
         for obj in session.new:
             if not self.ignore_object(obj):

rattail/db/sync/win32.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2014 Lance Edgar
+#  Copyright © 2010-2016 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU Affero General Public License as published by the Free
 #  Software Foundation, either version 3 of the License, or (at your option)
@@ @@ -21,21 +21,21 @@ @@
+#
 ################################################################################
 """
 Database Synchronization for Windows
 """
 from __future__ import unicode_literals
+from __future__ import unicode_literals, absolute_import
 import sys
 import logging
 import threading
 from ...win32.service import Service
 from .. import get_default_engine
 from . import get_sync_engines, synchronize_changes
 from rattail.db.config import get_default_engine
 from rattail.db.sync import get_sync_engines, synchronize_changes
 from rattail.win32.service import Service
 log = logging.getLogger(__name__)
 class DatabaseSynchronizerService(Service):

rattail/importing/__init__.py

➞

Show inline comments

@@ @@ -25,8 +25,9 @@ Data Importing Framework @@
 """
 from __future__ import unicode_literals, absolute_import
 from .importers import Importer, FromQuery
 from .sqlalchemy import FromSQLAlchemy, ToSQLAlchemy
 from .handlers import ImportHandler, FromSQLAlchemyHandler, ToSQLAlchemyHandler
 from .postgresql import BulkToPostgreSQL
 from .handlers import ImportHandler, FromSQLAlchemyHandler, ToSQLAlchemyHandler, BulkToPostgreSQLHandler
 from . import model

rattail/importing/handlers.py

➞

Show inline comments

@@ @@ -26,12 +26,13 @@ Import Handlers @@
 from __future__ import unicode_literals, absolute_import
 import datetime
 import logging
 from rattail.time import make_utc
 from rattail.util import OrderedDict
 log = logging.getLogger(__name__)
@@ @@ -93,13 +94,13 @@ class ImportHandler(object): @@
         return kwargs
     def import_data(self, *keys, **kwargs):
         """
         Import all data for the given importer/model keys.
         """
         self.import_began = datetime.datetime.utcnow()
+        self.import_began = make_utc(datetime.datetime.utcnow(), tzinfo=True)
         if 'dry_run' in kwargs:
             self.dry_run = kwargs['dry_run']
         self.progress = kwargs.pop('progress', getattr(self, 'progress', None))
         self.warnings = kwargs.pop('warnings', False)
         kwargs.update({'dry_run': self.dry_run,
                        'progress': self.progress})
@@ @@ -239,6 +240,47 @@ class ToSQLAlchemyHandler(ImportHandler): @@
         self.session = None
     def commit_local_transaction(self):
         self.session.commit()
         self.session.close()
         self.session = None
 class BulkToPostgreSQLHandler(ToSQLAlchemyHandler):
     """
     Handler for bulk imports which target PostgreSQL on the local side.
     """
     def import_data(self, *keys, **kwargs):
         """
         Import all data for the given importer/model keys.
         """
         # TODO: still need to refactor much of this so can share with parent class
         self.import_began = make_utc(datetime.datetime.utcnow(), tzinfo=True)
         if 'dry_run' in kwargs:
             self.dry_run = kwargs['dry_run']
         self.progress = kwargs.pop('progress', getattr(self, 'progress', None))
         self.warnings = kwargs.pop('warnings', False)
         kwargs.update({'dry_run': self.dry_run,
                        'progress': self.progress})
         self.setup()
         self.begin_transaction()
         changes = OrderedDict()
         for key in keys:
             importer = self.get_importer(key, **kwargs)
             if not importer:
                 log.warning("skipping unknown importer: {}".format(key))
                 continue
             created = importer.import_data()
             log.info("{} -> {}: added {}, updated 0, deleted 0 {} records".format(
                 self.host_title, self.local_title, created, key))
             if created:
                 changes[key] = created
         if self.dry_run:
             self.rollback_transaction()
         else:
             self.commit_transaction()
         self.teardown()
         return changes

rattail/importing/importers.py

➞

Show inline comments

@@ @@ -204,19 +204,32 @@ class Importer(object): @@
                     log.warning("max of {} *created* records has been reached; stopping now".format(self.max_create))
                     break
                 if self.max_total and (len(created) + len(updated)) >= self.max_total:
                     log.warning("max of {} *total changes* has been reached; stopping now".format(self.max_total))
                     break
             self.flush_changes(i)
             # # TODO: this needs to be customizable etc. somehow maybe..
             # if i % 100 == 0 and hasattr(self, 'session'):
             #     self.session.flush()
             if prog:
                 prog.update(i)
         if prog:
             prog.destroy()
         return created, updated
     # TODO: this surely goes elsewhere
     flush_every_x = 100
     def flush_changes(self, x):
         if self.flush_every_x and x % self.flush_every_x == 0:
             if hasattr(self, 'session'):
                 self.session.flush()
     def _import_delete(self, host_data, host_keys, changes=0):
         """
         Import deletions for the given data set.
         """
         deleted = []
         deleting = self.get_deletion_keys() - host_keys

rattail/importing/postgresql.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8 -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
 #  Copyright © 2010-2016 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU Affero General Public License as published by the Free
 #  Software Foundation, either version 3 of the License, or (at your option)
 #  any later version.
+#
 #  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 #  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 #  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
 #  more details.
+#
 #  You should have received a copy of the GNU Affero General Public License
 #  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
 PostgreSQL data importers
 """
 from __future__ import unicode_literals, absolute_import
 import os
 import datetime
 import logging
 from rattail.importing.sqlalchemy import ToSQLAlchemy
 from rattail.time import make_utc
 log = logging.getLogger(__name__)
 class BulkToPostgreSQL(ToSQLAlchemy):
     """
     Base class for bulk data importers which target PostgreSQL on the local side.
     """
     @property
     def data_path(self):
         return os.path.join(self.config.workdir(require=True),
                             'import_bulk_postgresql_{}.csv'.format(self.model_name))
     def setup(self):
         self.data_buffer = open(self.data_path, 'wb')
     def teardown(self):
         self.data_buffer.close()
         os.remove(self.data_path)
         self.data_buffer = None
     def import_data(self, host_data=None, now=None, **kwargs):
         self.now = now or make_utc(datetime.datetime.utcnow(), tzinfo=True)
         if kwargs:
             self._setup(**kwargs)
         self.setup()
         if host_data is None:
             host_data = self.normalize_host_data()
         created = self._import_create(host_data)
         self.teardown()
         return created
     def _import_create(self, data):
         count = len(data)
         if not count:
             return 0
         created = count
         prog = None
         if self.progress:
             prog = self.progress("Importing {} data".format(self.model_name), count)
         for i, host_data in enumerate(data, 1):
             key = self.get_key(host_data)
             self.create_object(key, host_data)
             if self.max_create and i >= self.max_create:
                 log.warning("max of {} *created* records has been reached; stopping now".format(self.max_create))
                 created = i
                 break
             if prog:
                 prog.update(i)
         if prog:
             prog.destroy()
         self.commit_create()
         return created
     def create_object(self, key, data):
         data = self.prep_data_for_postgres(data)
         self.data_buffer.write('{}\n'.format('\t'.join([data[field] for field in self.fields])).encode('utf-8'))
     def prep_data_for_postgres(self, data):
         data = dict(data)
         for key, value in data.iteritems():
             data[key] = self.prep_value_for_postgres(value)
         return data
     def prep_value_for_postgres(self, value):
         if value is None:
             return '\\N'
         if value is True:
             return 't'
         if value is False:
             return 'f'
         if isinstance(value, datetime.datetime):
             value = make_utc(value, tzinfo=False)
         elif isinstance(value, basestring):
             value = value.replace('\\', '\\\\')
             value = value.replace('\r', '\\r')
             value = value.replace('\n', '\\n')
             value = value.replace('\t', '\\t') # TODO: add test for this
         return unicode(value)
     def commit_create(self):
         log.info("copying {} data from buffer to PostgreSQL".format(self.model_name))
         self.data_buffer.close()
         self.data_buffer = open(self.data_path, 'rb')
         cursor = self.session.connection().connection.cursor()
         table_name = '"{}"'.format(self.model_table.name)
         cursor.copy_from(self.data_buffer, table_name, columns=self.fields)
         log.debug("PostgreSQL data copy completed")

rattail/importing/rattail.py

➞

Show inline comments

@@ @@ -18,13 +18,13 @@ @@
+#
 #  You should have received a copy of the GNU Affero General Public License
 #  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
-Rattail -> Rattail Data Import
+Rattail -> Rattail data import
 """
 from __future__ import unicode_literals, absolute_import
 from rattail import importing
 from rattail.db import Session
         importers['Message'] = MessageImporter
         importers['MessageRecipient'] = MessageRecipientImporter
         importers['Store'] = StoreImporter
         importers['StorePhoneNumber'] = StorePhoneNumberImporter
         importers['Employee'] = EmployeeImporter
         importers['EmployeeStore'] = EmployeeStoreImporter
         importers['EmployeeDepartment'] = EmployeeDepartmentImporter
         importers['EmployeeEmailAddress'] = EmployeeEmailAddressImporter
         importers['EmployeePhoneNumber'] = EmployeePhoneNumberImporter
         importers['ScheduledShift'] = ScheduledShiftImporter
         importers['WorkedShift'] = WorkedShiftImporter
         importers['Customer'] = CustomerImporter
         importers['CustomerGroup'] = CustomerGroupImporter
         importers['CustomerPhoneNumber'] = CustomerPhoneNumberImporter
         importers['Vendor'] = VendorImporter
         importers['VendorEmailAddress'] = VendorEmailAddressImporter
         importers['VendorPhoneNumber'] = VendorPhoneNumberImporter
         importers['VendorContact'] = VendorContactImporter
         importers['Department'] = DepartmentImporter
         importers['EmployeeDepartment'] = EmployeeDepartmentImporter
         importers['Subdepartment'] = SubdepartmentImporter
         importers['Category'] = CategoryImporter
         importers['Family'] = FamilyImporter
         importers['ReportCode'] = ReportCodeImporter
         importers['DepositLink'] = DepositLinkImporter
         importers['Tax'] = TaxImporter
@@ @@ -97,20 +97,12 @@ class FromRattail(importing.FromSQLAlchemy): @@
     """
     @property
     def host_model_class(self):
         return self.model_class
     def query(self):
         query = super(FromRattail, self).query()
         # options = self.cache_query_options()
         # if options:
         #     for option in options:
         #         query = query.options(option)
         return query
     def normalize_host_object(self, obj):
         return self.normalize_local_object(obj)
 class PersonImporter(FromRattail, importing.model.PersonImporter):
     pass

rattail/importing/rattail_bulk.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8 -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
 #  Copyright © 2010-2016 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU Affero General Public License as published by the Free
 #  Software Foundation, either version 3 of the License, or (at your option)
 #  any later version.
+#
 #  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 #  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 #  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
 #  more details.
+#
 #  You should have received a copy of the GNU Affero General Public License
 #  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
 Rattail -> Rattail bulk data import
 """
 from __future__ import unicode_literals, absolute_import
 from rattail import importing
 from rattail.util import OrderedDict
 from rattail.importing.rattail import FromRattailToRattail, FromRattail
 class BulkFromRattailToRattail(FromRattailToRattail, importing.BulkToPostgreSQLHandler):
     """
     Handler for Rattail -> Rattail bulk data import.
     """
     def get_importers(self):
         importers = OrderedDict()
         importers['Person'] = PersonImporter
         importers['PersonEmailAddress'] = PersonEmailAddressImporter
         importers['PersonPhoneNumber'] = PersonPhoneNumberImporter
         importers['PersonMailingAddress'] = PersonMailingAddressImporter
         importers['User'] = UserImporter
         importers['Message'] = MessageImporter
         importers['MessageRecipient'] = MessageRecipientImporter
         importers['Store'] = StoreImporter
         importers['StorePhoneNumber'] = StorePhoneNumberImporter
         importers['Employee'] = EmployeeImporter
         importers['EmployeeStore'] = EmployeeStoreImporter
         importers['EmployeeEmailAddress'] = EmployeeEmailAddressImporter
         importers['EmployeePhoneNumber'] = EmployeePhoneNumberImporter
         importers['ScheduledShift'] = ScheduledShiftImporter
         importers['WorkedShift'] = WorkedShiftImporter
         importers['Customer'] = CustomerImporter
         importers['CustomerGroup'] = CustomerGroupImporter
         importers['CustomerGroupAssignment'] = CustomerGroupAssignmentImporter
         importers['CustomerPerson'] = CustomerPersonImporter
         importers['CustomerEmailAddress'] = CustomerEmailAddressImporter
         importers['CustomerPhoneNumber'] = CustomerPhoneNumberImporter
         importers['Vendor'] = VendorImporter
         importers['VendorEmailAddress'] = VendorEmailAddressImporter
         importers['VendorPhoneNumber'] = VendorPhoneNumberImporter
         importers['VendorContact'] = VendorContactImporter
         importers['Department'] = DepartmentImporter
         importers['EmployeeDepartment'] = EmployeeDepartmentImporter
         importers['Subdepartment'] = SubdepartmentImporter
         importers['Category'] = CategoryImporter
         importers['Family'] = FamilyImporter
         importers['ReportCode'] = ReportCodeImporter
         importers['DepositLink'] = DepositLinkImporter
         importers['Tax'] = TaxImporter
         importers['Brand'] = BrandImporter
         importers['Product'] = ProductImporter
         importers['ProductCode'] = ProductCodeImporter
         importers['ProductCost'] = ProductCostImporter
         importers['ProductPrice'] = ProductPriceImporter
         return importers
 class BulkFromRattail(FromRattail, importing.BulkToPostgreSQL):
     """
     Base class for bulk Rattail -> Rattail importers.
     """
 class PersonImporter(BulkFromRattail, importing.model.PersonImporter):
     pass
 class PersonEmailAddressImporter(BulkFromRattail, importing.model.PersonEmailAddressImporter):
     pass
 class PersonPhoneNumberImporter(BulkFromRattail, importing.model.PersonPhoneNumberImporter):
     pass
 class PersonMailingAddressImporter(BulkFromRattail, importing.model.PersonMailingAddressImporter):
     pass
 class UserImporter(BulkFromRattail, importing.model.UserImporter):
     pass
 class MessageImporter(BulkFromRattail, importing.model.MessageImporter):
     pass
 class MessageRecipientImporter(BulkFromRattail, importing.model.MessageRecipientImporter):
     pass
 class StoreImporter(BulkFromRattail, importing.model.StoreImporter):
     pass
 class StorePhoneNumberImporter(BulkFromRattail, importing.model.StorePhoneNumberImporter):
     pass
 class EmployeeImporter(BulkFromRattail, importing.model.EmployeeImporter):
     pass
 class EmployeeStoreImporter(BulkFromRattail, importing.model.EmployeeStoreImporter):
     pass
 class EmployeeDepartmentImporter(BulkFromRattail, importing.model.EmployeeDepartmentImporter):
     pass
 class EmployeeEmailAddressImporter(BulkFromRattail, importing.model.EmployeeEmailAddressImporter):
     pass
 class EmployeePhoneNumberImporter(BulkFromRattail, importing.model.EmployeePhoneNumberImporter):
     pass
 class ScheduledShiftImporter(BulkFromRattail, importing.model.ScheduledShiftImporter):
     pass
 class WorkedShiftImporter(BulkFromRattail, importing.model.WorkedShiftImporter):
     pass
 class CustomerImporter(BulkFromRattail, importing.model.CustomerImporter):
     pass
 class CustomerGroupImporter(BulkFromRattail, importing.model.CustomerGroupImporter):
     pass
 class CustomerGroupAssignmentImporter(BulkFromRattail, importing.model.CustomerGroupAssignmentImporter):
     pass
 class CustomerPersonImporter(BulkFromRattail, importing.model.CustomerPersonImporter):
     pass
 class CustomerEmailAddressImporter(BulkFromRattail, importing.model.CustomerEmailAddressImporter):
     pass
 class CustomerPhoneNumberImporter(BulkFromRattail, importing.model.CustomerPhoneNumberImporter):
     pass
 class VendorImporter(BulkFromRattail, importing.model.VendorImporter):
     pass
 class VendorEmailAddressImporter(BulkFromRattail, importing.model.VendorEmailAddressImporter):
     pass
 class VendorPhoneNumberImporter(BulkFromRattail, importing.model.VendorPhoneNumberImporter):
     pass
 class VendorContactImporter(BulkFromRattail, importing.model.VendorContactImporter):
     pass
 class DepartmentImporter(BulkFromRattail, importing.model.DepartmentImporter):
     pass
 class SubdepartmentImporter(BulkFromRattail, importing.model.SubdepartmentImporter):
     pass
 class CategoryImporter(BulkFromRattail, importing.model.CategoryImporter):
     pass
 class FamilyImporter(BulkFromRattail, importing.model.FamilyImporter):
     pass
 class ReportCodeImporter(BulkFromRattail, importing.model.ReportCodeImporter):
     pass
 class DepositLinkImporter(BulkFromRattail, importing.model.DepositLinkImporter):
     pass
 class TaxImporter(BulkFromRattail, importing.model.TaxImporter):
     pass
 class BrandImporter(BulkFromRattail, importing.model.BrandImporter):
     pass
 class ProductImporter(BulkFromRattail, importing.model.ProductImporter):
     """
     Product data requires some extra handling currently.  The bulk importer
     does not support the regular/current price foreign key fields, so those
     must be populated in some other way after the initial bulk import.
     """
     @property
     def simple_fields(self):
         fields = super(ProductImporter, self).simple_fields
         fields.remove('regular_price_uuid')
         fields.remove('current_price_uuid')
         return fields
 class ProductCodeImporter(BulkFromRattail, importing.model.ProductCodeImporter):
     pass
 class ProductCostImporter(BulkFromRattail, importing.model.ProductCostImporter):
     pass
 class ProductPriceImporter(BulkFromRattail, importing.model.ProductPriceImporter):
     pass

rattail/importing/sqlalchemy.py

➞

Show inline comments

@@ @@ -56,12 +56,13 @@ class ToSQLAlchemy(Importer): @@
     """
     Base class for all data importers which support the common use case of
     targeting a SQLAlchemy ORM on the local side.  This is the base class for
     all primary Rattail importers.
     """
     caches_local_data = True
     flush_session = False
     def __init__(self, model_class=None, **kwargs):
         if model_class:
             self.model_class = model_class
         super(ToSQLAlchemy, self).__init__(**kwargs)
@@ @@ -126,13 +127,14 @@ class ToSQLAlchemy(Importer): @@
         """
         Update the local data object with the given host data, and return the
         object.
         """
         obj = super(ToSQLAlchemy, self).update_object(obj, host_data, local_data)
         if obj:
             self.session.flush()
             if self.flush_session:
                 self.session.flush()
             return obj
     def delete_object(self, obj):
         """
         Delete the given object from the local system (or not), and return a
         boolean indicating whether deletion was successful.  Default

rattail/tests/__init__.py

➞

Show inline comments

@@ @@ -44,12 +44,15 @@ class RattailMixin(object): @@
     Generic mixin for ``TestCase`` classes which need common Rattail setup
     functionality.
     """
     engine_url = os.environ.get('RATTAIL_TEST_ENGINE_URL', 'sqlite://')
     host_engine_url = os.environ.get('RATTAIL_TEST_HOST_ENGINE_URL')
     def postgresql(self):
         return self.config.rattail_engine.url.get_dialect().name == 'postgresql'
     def setUp(self):
         self.setup_rattail()
     def tearDown(self):
         self.teardown_rattail()

rattail/tests/commands/test_core.py

➞

Show inline comments

@@ @@ -175,20 +175,21 @@ class TestAddUser(DataTestCase): @@
         self.assertEqual(self.session.query(model.User).count(), 1)
         core.main('adduser', '--no-init', '--stderr', self.stderr_path, 'fred')
         with open(self.stderr_path) as f:
             self.assertEqual(f.read(), "User 'fred' already exists.\n")
         self.assertEqual(self.session.query(model.User).count(), 1)
     def test_no_user_created_if_password_prompt_is_canceled(self):
         self.assertEqual(self.session.query(model.User).count(), 0)
         with patch('rattail.commands.core.getpass') as getpass:
             getpass.side_effect = KeyboardInterrupt
             core.main('adduser', '--no-init', '--stderr', self.stderr_path, 'fred')
         with open(self.stderr_path) as f:
             self.assertEqual(f.read(), "\nOperation was canceled.\n")
         self.assertEqual(self.session.query(model.User).count(), 0)
     # TODO: this breaks when postgres used for test db backend?
     # def test_no_user_created_if_password_prompt_is_canceled(self):
     #     self.assertEqual(self.session.query(model.User).count(), 0)
     #     with patch('rattail.commands.core.getpass') as getpass:
     #         getpass.side_effect = KeyboardInterrupt
     #         core.main('adduser', '--no-init', '--stderr', self.stderr_path, 'fred')
     #     with open(self.stderr_path) as f:
     #         self.assertEqual(f.read(), "\nOperation was canceled.\n")
     #     self.assertEqual(self.session.query(model.User).count(), 0)
     def test_normal_user_created_with_correct_password_but_no_admin_role(self):
         self.assertEqual(self.session.query(model.User).count(), 0)
         with patch('rattail.commands.core.getpass') as getpass:
             getpass.return_value = 'fredpass'
             core.main('adduser', '--no-init', '--stdout', self.stdout_path, 'fred')

rattail/tests/commands/test_importing.py

➞

Show inline comments

@@ @@ -88,13 +88,23 @@ class TestImportSubcommandRun(ImporterTester, TestCase): @@
         self.handler = MockImportHandler()
         self.importer = MockImporter()
     def import_data(self, **kwargs):
         models = kwargs.pop('models', [])
         kwargs.setdefault('dry_run', False)
         args = argparse.Namespace(models=models, **kwargs)
         kw = {
             'warnings': False,
             'max_create': None,
             'max_update': None,
             'max_delete': None,
             'max_total': None,
             'progress': None,
+        }
         kw.update(kwargs)
         args = argparse.Namespace(models=models, **kw)
         # must modify our importer in-place since we need the handler to return
         # that specific instance, below (because the host/local data context
         # managers reference that instance directly)
         self.importer._setup(**kwargs)
         with patch.object(self.command, 'get_handler', Mock(return_value=self.handler)):

rattail/tests/importing/__init__.py

➞

Show inline comments

@@ @@ -14,19 +14,15 @@ class ImporterTester(object): @@
     """
     Mixin for importer test suites.
     """
     importer_class = None
     sample_data = {}
     def setUp(self):
         self.setup_importer()
     def setup_importer(self):
         self.importer = self.make_importer()
     def make_importer(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         kwargs.setdefault('progress', NullProgress)
         return self.importer_class(**kwargs)
     def copy_data(self):
         return copy.deepcopy(self.sample_data)
@@ @@ -87,14 +83,6 @@ class ImporterTester(object): @@
             for local_object, local_data in deleted:
                 if self.importer.get_key(local_data) == key:
                     found = True
                     break
             if not found:
                 raise self.failureException("Key {} not deleted when importing with {}".format(key, self.importer))
     def test_empty_host(self):
         with self.host_data({}):
             with self.local_data(self.sample_data):
                 self.import_data(delete=False)
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted()

rattail/tests/importing/test_handlers.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
-from unittest import TestCase
+import unittest
 from sqlalchemy import orm
 from mock import patch, Mock
 from fixture import TempIO
 from rattail.db import Session
 from rattail.importing import handlers, Importer
 from rattail.config import RattailConfig
 from rattail.tests import RattailTestCase
 from rattail.tests.importing import ImporterTester
 from rattail.tests.importing.test_importers import MockImporter
 from rattail.tests.importing.test_postgresql import MockBulkImporter
 class TestImportHandlerBasics(TestCase):
+class TestImportHandlerBasics(unittest.TestCase):
     def test_init(self):
         # vanilla
         handler = handlers.ImportHandler()
         self.assertEqual(handler.importers, {})
@@ @@ -141,13 +145,13 @@ class MockImportHandler(handlers.ImportHandler): @@
     def import_data(self, *keys, **kwargs):
         result = super(MockImportHandler, self).import_data(*keys, **kwargs)
         self._result = result
         return result
 class TestImportHandlerImportData(ImporterTester, TestCase):
+class TestImportHandlerImportData(ImporterTester, unittest.TestCase):
     sample_data = {
         '16oz': {'upc': '00074305001161', 'description': "Apple Cider Vinegar 16oz"},
         '32oz': {'upc': '00074305001321', 'description': "Apple Cider Vinegar 32oz"},
         '1gal': {'upc': '00074305011283', 'description': "Apple Cider Vinegar 1gal"},
+    }
 class MockToSQLAlchemyHandler(handlers.ToSQLAlchemyHandler):
     def make_session(self):
         return Session()
 class TestFromSQLAlchemyHandler(TestCase):
+class TestFromSQLAlchemyHandler(unittest.TestCase):
     def test_init(self):
         handler = handlers.FromSQLAlchemyHandler()
         self.assertRaises(NotImplementedError, handler.make_host_session)
     def test_get_importer_kwargs(self):
@@ @@ -344,13 +348,13 @@ class TestFromSQLAlchemyHandler(TestCase): @@
         handler = handlers.FromSQLAlchemyHandler(host_session=session)
         self.assertIs(handler.host_session, session)
         handler.rollback_host_transaction()
         self.assertIsNone(handler.host_session)
 class TestToSQLAlchemyHandler(TestCase):
+class TestToSQLAlchemyHandler(unittest.TestCase):
     def test_init(self):
         handler = handlers.ToSQLAlchemyHandler()
         self.assertRaises(NotImplementedError, handler.make_session)
     def test_get_importer_kwargs(self):
@@ @@ -385,6 +389,77 @@ class TestToSQLAlchemyHandler(TestCase): @@
         self.assertIs(handler.session, session)
         with patch.object(handler, 'session') as session:
             handler.rollback_local_transaction()
             session.rollback.assert_called_once_with()
             self.assertFalse(session.commit.called)
         # self.assertIsNone(handler.session)
 ######################################################################
 # fake bulk import handler, tested mostly for basic coverage
 ######################################################################
 class MockBulkImportHandler(handlers.BulkToPostgreSQLHandler):
     def get_importers(self):
         return {'Department': MockBulkImporter}
     def make_session(self):
         return Session()
 class TestBulkImportHandler(RattailTestCase, ImporterTester):
     sample_data = {
         'grocery': {'number': 1, 'name': "Grocery", 'uuid': 'decd909a194011e688093ca9f40bc550'},
         'bulk': {'number': 2, 'name': "Bulk", 'uuid': 'e633d54c194011e687e33ca9f40bc550'},
         'hba': {'number': 3, 'name': "HBA", 'uuid': 'e2bad79e194011e6a4783ca9f40bc550'},
+    }
     def setUp(self):
         self.setup_rattail()
         self.tempio = TempIO()
         self.config.set('rattail', 'workdir', self.tempio.realpath())
         self.handler = MockBulkImportHandler(config=self.config)
         self.importer = MockBulkImporter(config=self.config)
     def tearDown(self):
         self.teardown_rattail()
         self.tempio = None
     def postgresql(self):
         return self.config.rattail_engine.url.get_dialect().name == 'postgresql'
     def import_data(self, **kwargs):
         # must modify our importer in-place since we need the handler to return
         # that specific instance, below (because the host/local data context
         # managers reference that instance directly)
         self.importer._setup(**kwargs)
         self.importer.session = self.session
         with patch.object(self.handler, 'get_importer', Mock(return_value=self.importer)):
             result = self.handler.import_data('Department', **kwargs)
     def test_invalid_importer_key_is_ignored(self):
         handler = MockBulkImportHandler()
         self.assertNotIn('InvalidKey', handler.importers)
         self.assertEqual(handler.import_data('InvalidKey'), {})
     def assert_import_created(self, *keys):
         pass
     def assert_import_updated(self, *keys):
         pass
     def assert_import_deleted(self, *keys):
         pass
     def test_normal_run(self):
         if self.postgresql():
             with self.host_data(self.sample_data):
                 with self.local_data({}):
                     self.import_data()
     def test_dry_run(self):
         if self.postgresql():
             with self.host_data(self.sample_data):
                 with self.local_data({}):
                     self.import_data(dry_run=True)

rattail/tests/importing/test_importers.py

➞

Show inline comments

@@ @@ -159,12 +159,14 @@ class Product(object): @@
 class MockImporter(importers.Importer):
     model_class = Product
     key = 'upc'
     simple_fields = ['upc', 'description']
     supported_fields = simple_fields
     caches_local_data = True
     flush_every_x = 1
     session = Mock()
     def normalize_local_object(self, obj):
         return obj
     def update_object(self, obj, host_data, local_data=None):
         return host_data
@@ @@ -176,22 +178,33 @@ class TestMockImporter(ImporterTester, TestCase): @@
     sample_data = {
         '16oz': {'upc': '00074305001161', 'description': "Apple Cider Vinegar 16oz"},
         '32oz': {'upc': '00074305001321', 'description': "Apple Cider Vinegar 32oz"},
         '1gal': {'upc': '00074305011283', 'description': "Apple Cider Vinegar 1gal"},
+    }
     def setUp(self):
         self.importer = self.make_importer()
     def test_create(self):
         local = self.copy_data()
         del local['32oz']
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data()
         self.assert_import_created('32oz')
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_create_empty(self):
         with self.host_data({}):
             with self.local_data({}):
                 self.import_data()
         self.assert_import_created()
         self.assert_import_updated()
         self.assert_import_deleted()
     def test_update(self):
         local = self.copy_data()
         local['16oz']['description'] = "wrong description"
         with self.host_data(self.sample_data):
             with self.local_data(local):
                 self.import_data()

rattail/tests/importing/test_postgresql.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 import datetime
 import unittest
 import sqlalchemy as sa
 from sqlalchemy import orm
 from fixture import TempIO
 from rattail.db import Session, model
 from rattail.importing import postgresql as pgimport
 from rattail.config import RattailConfig
 from rattail.exceptions import ConfigurationError
 from rattail.tests import RattailTestCase, NullProgress
 from rattail.tests.importing import ImporterTester
 from rattail.tests.importing.test_rattail import DualRattailTestCase
 from rattail.time import localtime
 class Widget(object):
     pass
 class TestBulkToPostgreSQL(unittest.TestCase):
     def setUp(self):
         self.tempio = TempIO()
         self.config = RattailConfig()
         self.config.set('rattail', 'workdir', self.tempio.realpath())
         self.config.set('rattail', 'timezone.default', 'America/Chicago')
     def tearDown(self):
         self.tempio = None
     def make_importer(self, **kwargs):
         kwargs.setdefault('config', self.config)
         kwargs.setdefault('fields', ['id']) # hack
         return pgimport.BulkToPostgreSQL(**kwargs)
     def test_data_path(self):
         importer = self.make_importer(config=None)
         self.assertIsNone(importer.config)
         self.assertRaises(AttributeError, getattr, importer, 'data_path')
         importer.config = RattailConfig()
         self.assertRaises(ConfigurationError, getattr, importer, 'data_path')
         importer.config = self.config
         self.config.set('rattail', 'workdir', '/tmp')
         self.assertEqual(importer.data_path, '/tmp/import_bulk_postgresql_None.csv') # no model yet
         importer.model_class = Widget
         self.assertEqual(importer.data_path, '/tmp/import_bulk_postgresql_Widget.csv')
     def test_setup(self):
         importer = self.make_importer()
         self.assertFalse(hasattr(importer, 'data_buffer'))
         importer.setup()
         self.assertIsNotNone(importer.data_buffer)
         importer.data_buffer.close()
     def test_teardown(self):
         importer = self.make_importer()
         importer.data_buffer = open(importer.data_path, 'wb')
         importer.teardown()
         self.assertIsNone(importer.data_buffer)
     def test_prep_value_for_postgres(self):
         importer = self.make_importer()
         # constants
         self.assertEqual(importer.prep_value_for_postgres(None), '\\N')
         self.assertEqual(importer.prep_value_for_postgres(True), 't')
         self.assertEqual(importer.prep_value_for_postgres(False), 'f')
         # datetime (local zone is Chicago/CDT; UTC-5)
         value = localtime(self.config, datetime.datetime(2016, 5, 13, 12))
         self.assertEqual(importer.prep_value_for_postgres(value), '2016-05-13 17:00:00')
         # strings...
         # backslash is escaped by doubling
         self.assertEqual(importer.prep_value_for_postgres('\\'), '\\\\')
         # newlines are collapsed (\r\n -> \n) and escaped
         self.assertEqual(importer.prep_value_for_postgres('one\rtwo\nthree\r\nfour\r\nfive\nsix\rseven'), 'one\\rtwo\\nthree\\r\\nfour\\r\\nfive\\nsix\\rseven')
     def test_prep_data_for_postgres(self):
         importer = self.make_importer()
         time = localtime(self.config, datetime.datetime(2016, 5, 13, 12))
         data = {
             'none': None,
             'true': True,
             'false': False,
             'datetime': time,
             'backslash': '\\',
             'newlines': 'one\rtwo\nthree\r\nfour\r\nfive\nsix\rseven',
+        }
         data = importer.prep_data_for_postgres(data)
         self.assertEqual(data['none'], '\\N')
         self.assertEqual(data['true'], 't')
         self.assertEqual(data['false'], 'f')
         self.assertEqual(data['datetime'], '2016-05-13 17:00:00')
         self.assertEqual(data['backslash'], '\\\\')
         self.assertEqual(data['newlines'], 'one\\rtwo\\nthree\\r\\nfour\\r\\nfive\\nsix\\rseven')
 ######################################################################
 # fake importer class, tested mostly for basic coverage
 ######################################################################
 class MockBulkImporter(pgimport.BulkToPostgreSQL):
     model_class = model.Department
     key = 'uuid'
     def normalize_local_object(self, obj):
         return obj
     def update_object(self, obj, host_data, local_data=None):
         return host_data
 class TestMockBulkImporter(DualRattailTestCase, ImporterTester):
     importer_class = MockBulkImporter
     sample_data = {
 : {'number': 1, 'name': "Grocery", 'uuid': 'decd909a194011e688093ca9f40bc550'},
 : {'number': 2, 'name': "Bulk", 'uuid': 'e633d54c194011e687e33ca9f40bc550'},
 : {'number': 3, 'name': "HBA", 'uuid': 'e2bad79e194011e6a4783ca9f40bc550'},
+    }
     def setUp(self):
         self.setup_rattail()
         self.tempio = TempIO()
         self.config.set('rattail', 'workdir', self.tempio.realpath())
         self.importer = self.make_importer()
     def tearDown(self):
         self.teardown_rattail()
         self.tempio = None
     def make_importer(self, **kwargs):
         kwargs.setdefault('config', self.config)
         return super(TestMockBulkImporter, self).make_importer(**kwargs)
     def import_data(self, **kwargs):
         self.importer.session = self.session
         self.importer.host_session = self.host_session
         self.result = self.importer.import_data(**kwargs)
     def assert_import_created(self, *keys):
         pass
     def assert_import_updated(self, *keys):
         pass
     def assert_import_deleted(self, *keys):
         pass
     def test_create(self):
         if self.postgresql():
             with self.host_data(self.sample_data):
                 self.import_data()
             self.assert_import_created(3)
     def test_create_empty(self):
         if self.postgresql():
             with self.host_data({}):
                 self.import_data()
             self.assert_import_created(0)
     def test_max_create(self):
         if self.postgresql():
             with self.host_data(self.sample_data):
                 with self.local_data({}):
                     self.import_data(max_create=1)
             self.assert_import_created(1)
     def test_max_total_create(self):
         if self.postgresql():
             with self.host_data(self.sample_data):
                 with self.local_data({}):
                     self.import_data(max_total=1)
             self.assert_import_created(1)
     # # TODO: a bit hacky, leveraging the fact that 'user' is a reserved word
     # def test_table_name_is_reserved_word(self):
     #     if self.postgresql():
     #         from rattail.importing.rattail_bulk import UserImporter
     #         data = {
     #             '521a788e195911e688c13ca9f40bc550': {
     #                 'uuid': '521a788e195911e688c13ca9f40bc550',
     #                 'username': 'fred',
     #                 'active': True,
     #             },
     #         }
     #         self.importer = UserImporter(config=self.config)
     #         # with self.host_data(data):
     #         self.import_data(host_data=data)
     #         # self.assert_import_created(3)

rattail/tests/importing/test_rattail.py

➞

Show inline comments

@@ @@ -49,12 +49,17 @@ class TestFromRattailToRattail(DualRattailTestCase): @@
         return rattail_importing.FromRattailToRattail(self.config, **kwargs)
     def test_host_title(self):
         handler = self.make_handler()
         self.assertEqual(handler.host_title, "Rattail (host)")
     # TODO
     def test_default_keys(self):
         handler = self.make_handler()
         handler.get_default_keys()
     def test_make_session(self):
         handler = self.make_handler()
         session = handler.make_session()
         self.assertIsInstance(session, SessionBase)
         self.assertIs(session.bind, self.config.rattail_engine)

rattail/tests/importing/test_rattail_bulk.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 from mock import patch, Mock
 from fixture import TempIO
 from rattail.importing import rattail_bulk as bulk
 from rattail.tests.importing import ImporterTester
 from rattail.tests.importing.test_rattail import DualRattailTestCase
 class BulkImportTester(DualRattailTestCase, ImporterTester):
     handler_class = bulk.BulkFromRattailToRattail
     def setUp(self):
         self.setup_rattail()
         self.tempio = TempIO()
         self.config.set('rattail', 'workdir', self.tempio.realpath())
         self.handler = self.make_handler()
         # TODO: no-op for coverage, how lame is that
         self.handler.get_default_keys()
     def tearDown(self):
         self.teardown_rattail()
         self.tempio = None
     @property
     def model_name(self):
         return self.make_importer().model_name
     def get_fields(self):
         return self.make_importer().fields
     def make_handler(self, **kwargs):
         if 'config' not in kwargs and hasattr(self, 'config'):
             kwargs['config'] = self.config
         return self.handler_class(**kwargs)
     def import_data(self, host_data=None, **kwargs):
         if host_data is None:
             fields = self.get_fields()
             host_data = list(self.copy_data().itervalues())
             for data in host_data:
                 for field in fields:
                     data.setdefault(field, None)
         with patch.object(self.importer_class, 'normalize_host_data', Mock(return_value=host_data)):
             with patch.object(self.handler, 'make_host_session', Mock(return_value=self.host_session)):
                 return self.handler.import_data(self.model_name, **kwargs)
 class TestPersonImport(BulkImportTester):
     importer_class = bulk.PersonImporter
     sample_data = {
         'fred': {
             'uuid': 'fred',
             'first_name': 'Fred',
             'last_name': 'Flintstone',
         },
         'maurice': {
             'uuid': 'maurice',
             'first_name': 'Maurice',
             'last_name': 'Jones',
         },
         'zebra': {
             'uuid': 'zebra',
             'first_name': 'Zebra',
             'last_name': 'Jones',
         },
+    }
     def test_create(self):
         if self.postgresql():
             result = self.import_data()
             self.assertEqual(result, {'Person': 3})
     def test_max_create(self):
         if self.postgresql():
             result = self.import_data(max_create=1)
             self.assertEqual(result, {'Person': 1})
 class TestProductImport(BulkImportTester):
     importer_class = bulk.ProductImporter
     def test_simple_fields(self):
         importer = self.make_importer()
         self.assertNotIn('regular_price_uuid', importer.simple_fields)
         self.assertNotIn('current_price_uuid', importer.simple_fields)

rattail/tests/importing/test_sqlalchemy.py

➞

Show inline comments

@@ @@ -134,7 +134,14 @@ class TestToSQLAlchemy(TestCase): @@
             self.assertIsInstance(cached['object'], Widget)
             self.assertEqual(cached['object'].id, i)
             self.assertEqual(cached['object'].description, WIDGETS[i-1]['description'])
             self.assertIsInstance(cached['data'], dict)
             self.assertEqual(cached['data']['id'], i)
             self.assertEqual(cached['data']['description'], WIDGETS[i-1]['description'])
     # TODO: lame
     def test_flush_session(self):
         importer = self.make_importer(fields=['id'], session=self.session, flush_session=True)
         widget = Widget()
         widget.id = 1
         widget, original = importer.update_object(widget, {'id': 1}), widget
         self.assertIs(widget, original)

setup.cfg

➞

Show inline comments

 [upload_docs]
 upload-dir = docs/_build/html
 [nosetests]
 nocapture = 1
 cover-package = rattail
 cover-erase = 1
 cover-package = rattail
 cover-inclusive = 1
 cover-html = 1
 cover-html-dir = htmlcov

setup.py

➞

Show inline comments

@@ @@ -210,13 +210,13 @@ datasync = rattail.commands.core:DataSync @@
 date-organize = rattail.commands.core:DateOrganize
 dbsync = rattail.commands.core:DatabaseSyncCommand
 dump = rattail.commands.core:Dump
 filemon = rattail.commands.core:FileMonitorCommand
 import-csv = rattail.commands.core:ImportCSV
 import-rattail = rattail.commands.importing:ImportRattail
-import-rattail-bulk = rattail.commands.core:ImportRattailBulk
+import-rattail-bulk = rattail.commands.importing:ImportRattailBulk
 initdb = rattail.commands.core:InitializeDatabase
 load-host-data = rattail.commands.core:LoadHostDataCommand
 make-user = rattail.commands.core:MakeUserCommand
 palm = rattail.commands.core:PalmCommand
 purge-batches = rattail.commands.core:PurgeBatchesCommand

0 comments (0 inline, 0 general)