Changeset - 437ecfe6dd16
Parent rev.
Child rev.
[Not reviewed]
0 9 0
Lance Edgar - 8 years ago 2016-05-16 15:32:15
ledgar@sacfoodcoop.com
More tweaks for new importer framework

* Fix bug where some args weren't passed from command to handler
* Add new `ToRattailHandler` for convenience
* Add `ImportHandler.commit_partial_host` flag and logic
* Add `Importer.empty_local_data` flag and logic
* Fix bug where `Importer.delete` flag was ON by default
* tests for hopefully everything relevant..
9 files changed with 257 insertions and 83 deletions:
covered.cfg
2
17
rattail/commands/importing.py
3
rattail/importing/__init__.py
1
rattail/importing/handlers.py
14
6
rattail/importing/importers.py
3
1
rattail/importing/rattail.py
54
45
rattail/tests/commands/test_importing.py
7
4
rattail/tests/importing/test_handlers.py
139
7
rattail/tests/importing/test_importers.py
34
3
0 comments (0 inline, 0 general)
covered.cfg
➞
Show inline comments
 

			




	
	
	
		
 
[nosetests]

			




	
	
	
		
 
nocapture = 1

			




	
	
	
		
 
tests = rattail.tests.test_barcodes,

			




	
	
	
		
 
        rattail.tests.commands.test_importing,

			




	
	
	
		
 
        rattail.tests.db.test_core,

			




	
	
	
		
 
        rattail.tests.db.model.test_core,

			




	
	
	
		
 
        rattail.tests.db.model.test_customers,

			




	
	
	
		
 
        rattail.tests.db.model.test_datasync,

			




	
	
	
		
 
        rattail.tests.db.model.test_org,

			




	
	
	
		
 
        rattail.tests.db.model.test_people,

			




	
	
	
		
 
tests = rattail.tests.commands.test_importing,

			




	
	
	
		
 
        rattail.tests.filemon.test_actions,

			




	
	
	
		
 
        rattail.tests.filemon.test_config,

			




	
	
	
		
 
        rattail.tests.filemon.test_util,

			




	
	
	
		
 
        rattail.tests.importing

			




	
	
	
		
 
with-coverage = 1

			




	
	
	
		
 
cover-erase = 1

			




	
	
	
		
 
cover-package = rattail.barcodes,

			




	
	
	
		
 
        rattail.commands.importing,

			




	
	
	
		
 
        rattail.db.core,

			




	
	
	
		
 
        rattail.db.model.core,

			




	
	
	
		
 
        rattail.db.model.customers,

			




	
	
	
		
 
        rattail.db.model.datasync,

			




	
	
	
		
 
        rattail.db.model.org,

			




	
	
	
		
 
        rattail.db.model.people,

			




	
	
	
		
 
        rattail.enum,

			




	
	
	
		
 
cover-package = rattail.commands.importing,

			




	
	
	
		
 
        rattail.filemon.actions,

			




	
	
	
		
 
        rattail.filemon.config,

			




	
	
	
		
 
        rattail.filemon.util,

			




        

    


    
    

    

        

                

                    rattail/commands/importing.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -161,8 +161,11 @@ class ImportSubcommand(Subcommand):

			




	
	
	
		
 
        kwargs = {

			




	
	
	
		
 
            'dry_run': args.dry_run,

			




	
	
	
		
 
            'warnings': args.warnings,

			




	
	
	
		
 
            'create': args.create,

			




	
	
	
		
 
            'max_create': args.max_create,

			




	
	
	
		
 
            'update': args.update,

			




	
	
	
		
 
            'max_update': args.max_update,

			




	
	
	
		
 
            'delete': args.delete,

			




	
	
	
		
 
            'max_delete': args.max_delete,

			




	
	
	
		
 
            'max_total': args.max_total,

			




	
	
	
		
 
            'progress': self.progress,

			




        

    


    
    

    

        

                

                    rattail/importing/__init__.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -30,4 +30,5 @@ from .importers import Importer, FromQuery

			




	
	
	
		
 
from .sqlalchemy import FromSQLAlchemy, ToSQLAlchemy

			




	
	
	
		
 
from .postgresql import BulkToPostgreSQL

			




	
	
	
		
 
from .handlers import ImportHandler, FromSQLAlchemyHandler, ToSQLAlchemyHandler, BulkToPostgreSQLHandler

			




	
	
	
		
 
from .rattail import ToRattailHandler

			




	
	
	
		
 
from . import model

			




        

    


    
    

    

        

                

                    rattail/importing/handlers.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -51,6 +51,7 @@ class ImportHandler(object):

			




	
	
	
		
 
    local_title = None

			




	
	
	
		
 
    progress = None

			




	
	
	
		
 
    dry_run = False

			




	
	
	
		
 
    commit_host_partial = False

			




	
	
	
		
 

			




	
	
	
		
 
    def __init__(self, config=None, **kwargs):

			




	
	
	
		
 
        self.config = config

			




	
	
		
 
@@ -116,27 +117,33 @@ class ImportHandler(object):

			




	
	
	
		
 
        self.begin_transaction()

			




	
	
	
		
 
        changes = OrderedDict()

			




	
	
	
		
 

			




	
	
	
		
 
        try:

			




	
	
	
		
 
            for key in keys:

			




	
	
	
		
 
                importer = self.get_importer(key, **kwargs)

			




	
	
	
		
 
            if not importer:

			




	
	
	
		
 
                log.warning("skipping unknown importer: {}".format(key))

			




	
	
	
		
 
                continue

			




	
	
	
		
 

			




	
	
	
		
 
                if importer:

			




	
	
	
		
 
                    created, updated, deleted = importer.import_data()

			




	
	
	
		
 

			




	
	
	
		
 
                    changed = bool(created or updated or deleted)

			




	
	
	
		
 
                    logger = log.warning if changed and self.warnings else log.info

			




	
	
	
		
 
                    logger("{} -> {}: added {}, updated {}, deleted {} {} records".format(

			




	
	
	
		
 
                        self.host_title, self.local_title, len(created), len(updated), len(deleted), key))

			




	
	
	
		
 
                    if changed:

			




	
	
	
		
 
                        changes[key] = created, updated, deleted

			




	
	
	
		
 

			




	
	
	
		
 
                else:

			




	
	
	
		
 
                    log.warning("skipping unknown importer: {}".format(key))

			




	
	
	
		
 
        except:

			




	
	
	
		
 
            if self.commit_host_partial and not self.dry_run:

			




	
	
	
		
 
                log.warning("{host} -> {local}: committing partial transaction on host {host} (despite error)".format(

			




	
	
	
		
 
                    host=self.host_title, local=self.local_title))

			




	
	
	
		
 
                self.commit_host_transaction()

			




	
	
	
		
 
            raise

			




	
	
	
		
 
        else:

			




	
	
	
		
 
            if changes:

			




	
	
	
		
 
                self.process_changes(changes)

			




	
	
	
		
 
            if self.dry_run:

			




	
	
	
		
 
                self.rollback_transaction()

			




	
	
	
		
 
            else:

			




	
	
	
		
 
                self.commit_transaction()

			




	
	
	
		
 

			




	
	
	
		
 
        self.teardown()

			




	
	
	
		
 
        return changes

			




	
	
	
		
 

			




	
	
		
 
@@ -219,6 +226,7 @@ class ImportHandler(object):

			




	
	
	
		
 
            key = 'rattail_import_updates'

			




	
	
	
		
 

			




	
	
	
		
 
        send_email(self.config, key, fallback_key='rattail_import_updates', data=data)

			




	
	
	
		
 
        log.info("warning email was sent for {} -> {} import".format(self.host_title, self.local_title))

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class FromSQLAlchemyHandler(ImportHandler):

			




        

    


    
    

    

        

                

                    rattail/importing/importers.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -67,6 +67,7 @@ class Importer(object):

			




	
	
	
		
 
    max_total = None

			




	
	
	
		
 
    progress = None

			




	
	
	
		
 

			




	
	
	
		
 
    empty_local_data = False

			




	
	
	
		
 
    caches_local_data = False

			




	
	
	
		
 
    cached_local_data = None

			




	
	
	
		
 

			




	
	
		
 
@@ -90,7 +91,7 @@ class Importer(object):

			




	
	
	
		
 
    def _setup(self, **kwargs):

			




	
	
	
		
 
        self.create = kwargs.pop('create', self.allow_create) and self.allow_create

			




	
	
	
		
 
        self.update = kwargs.pop('update', self.allow_update) and self.allow_update

			




	
	
	
		
 
        self.delete = kwargs.pop('delete', self.allow_delete) and self.allow_delete

			




	
	
	
		
 
        self.delete = kwargs.pop('delete', False) and self.allow_delete

			




	
	
	
		
 
        for key, value in kwargs.iteritems():

			




	
	
	
		
 
            setattr(self, key, value)

			




	
	
	
		
 

			




	
	
		
 
@@ -354,6 +355,7 @@ class Importer(object):

			




	
	
	
		
 
        in effect, otherwise return the value from

			




	
	
	
		
 
        :meth:`get_single_local_object()`.

			




	
	
	
		
 
        """

			




	
	
	
		
 
        if not self.empty_local_data:

			




	
	
	
		
 
            if self.caches_local_data and self.cached_local_data is not None:

			




	
	
	
		
 
                data = self.cached_local_data.get(key)

			




	
	
	
		
 
                return data['object'] if data else None

			




        

    


    
    

    

        

                

                    rattail/importing/rattail.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -26,12 +26,24 @@ Rattail -> Rattail data import

			




	
	
	
		
 

			




	
	
	
		
 
from __future__ import unicode_literals, absolute_import

			




	
	
	
		
 

			




	
	
	
		
 
from rattail import importing

			




	
	
	
		
 
from rattail.db import Session

			




	
	
	
		
 
from rattail.importing import model

			




	
	
	
		
 
from rattail.importing.handlers import FromSQLAlchemyHandler, ToSQLAlchemyHandler

			




	
	
	
		
 
from rattail.importing.sqlalchemy import FromSQLAlchemy

			




	
	
	
		
 
from rattail.util import OrderedDict

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class FromRattailToRattail(importing.FromSQLAlchemyHandler, importing.ToSQLAlchemyHandler):

			




	
	
	
		
 
class ToRattailHandler(ToSQLAlchemyHandler):

			




	
	
	
		
 
    """

			




	
	
	
		
 
    Base class for import handlers which target a Rattail database on the local side.

			




	
	
	
		
 
    """

			




	
	
	
		
 
    local_title = "Rattail"

			




	
	
	
		
 

			




	
	
	
		
 
    def make_session(self):

			




	
	
	
		
 
        return Session()

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class FromRattailToRattail(FromSQLAlchemyHandler, ToRattailHandler):

			




	
	
	
		
 
    """

			




	
	
	
		
 
    Handler for Rattail -> Rattail data import.

			




	
	
	
		
 
    """

			




	
	
		
 
@@ -42,9 +54,6 @@ class FromRattailToRattail(importing.FromSQLAlchemyHandler, importing.ToSQLAlche

			




	
	
	
		
 
    def host_title(self):

			




	
	
	
		
 
        return "Rattail ({})".format(self.dbkey)

			




	
	
	
		
 

			




	
	
	
		
 
    def make_session(self):

			




	
	
	
		
 
        return Session()

			




	
	
	
		
 

			




	
	
	
		
 
    def make_host_session(self):

			




	
	
	
		
 
        return Session(bind=self.config.rattail_engines[self.dbkey])

			




	
	
	
		
 

			




	
	
		
 
@@ -98,7 +107,7 @@ class FromRattailToRattail(importing.FromSQLAlchemyHandler, importing.ToSQLAlche

			




	
	
	
		
 
        return keys

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class FromRattail(importing.FromSQLAlchemy):

			




	
	
	
		
 
class FromRattail(FromSQLAlchemy):

			




	
	
	
		
 
    """

			




	
	
	
		
 
    Base class for Rattail -> Rattail data importers.

			




	
	
	
		
 
    """

			




	
	
		
 
@@ -111,22 +120,22 @@ class FromRattail(importing.FromSQLAlchemy):

			




	
	
	
		
 
        return self.normalize_local_object(obj)

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class PersonImporter(FromRattail, importing.model.PersonImporter):

			




	
	
	
		
 
class PersonImporter(FromRattail, model.PersonImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class PersonEmailAddressImporter(FromRattail, importing.model.PersonEmailAddressImporter):

			




	
	
	
		
 
class PersonEmailAddressImporter(FromRattail, model.PersonEmailAddressImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class PersonPhoneNumberImporter(FromRattail, importing.model.PersonPhoneNumberImporter):

			




	
	
	
		
 
class PersonPhoneNumberImporter(FromRattail, model.PersonPhoneNumberImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class PersonMailingAddressImporter(FromRattail, importing.model.PersonMailingAddressImporter):

			




	
	
	
		
 
class PersonMailingAddressImporter(FromRattail, model.PersonMailingAddressImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class UserImporter(FromRattail, importing.model.UserImporter):

			




	
	
	
		
 
class UserImporter(FromRattail, model.UserImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class AdminUserImporter(FromRattail, importing.model.AdminUserImporter):

			




	
	
	
		
 
class AdminUserImporter(FromRattail, model.AdminUserImporter):

			




	
	
	
		
 

			




	
	
	
		
 
    def normalize_host_object(self, user):

			




	
	
	
		
 
        data = super(AdminUserImporter, self).normalize_local_object(user) # sic

			




	
	
		
 
@@ -135,101 +144,101 @@ class AdminUserImporter(FromRattail, importing.model.AdminUserImporter):

			




	
	
	
		
 
        return data

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class MessageImporter(FromRattail, importing.model.MessageImporter):

			




	
	
	
		
 
class MessageImporter(FromRattail, model.MessageImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class MessageRecipientImporter(FromRattail, importing.model.MessageRecipientImporter):

			




	
	
	
		
 
class MessageRecipientImporter(FromRattail, model.MessageRecipientImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class StoreImporter(FromRattail, importing.model.StoreImporter):

			




	
	
	
		
 
class StoreImporter(FromRattail, model.StoreImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class StorePhoneNumberImporter(FromRattail, importing.model.StorePhoneNumberImporter):

			




	
	
	
		
 
class StorePhoneNumberImporter(FromRattail, model.StorePhoneNumberImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class EmployeeImporter(FromRattail, importing.model.EmployeeImporter):

			




	
	
	
		
 
class EmployeeImporter(FromRattail, model.EmployeeImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class EmployeeStoreImporter(FromRattail, importing.model.EmployeeStoreImporter):

			




	
	
	
		
 
class EmployeeStoreImporter(FromRattail, model.EmployeeStoreImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class EmployeeDepartmentImporter(FromRattail, importing.model.EmployeeDepartmentImporter):

			




	
	
	
		
 
class EmployeeDepartmentImporter(FromRattail, model.EmployeeDepartmentImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class EmployeeEmailAddressImporter(FromRattail, importing.model.EmployeeEmailAddressImporter):

			




	
	
	
		
 
class EmployeeEmailAddressImporter(FromRattail, model.EmployeeEmailAddressImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class EmployeePhoneNumberImporter(FromRattail, importing.model.EmployeePhoneNumberImporter):

			




	
	
	
		
 
class EmployeePhoneNumberImporter(FromRattail, model.EmployeePhoneNumberImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class ScheduledShiftImporter(FromRattail, importing.model.ScheduledShiftImporter):

			




	
	
	
		
 
class ScheduledShiftImporter(FromRattail, model.ScheduledShiftImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class WorkedShiftImporter(FromRattail, importing.model.WorkedShiftImporter):

			




	
	
	
		
 
class WorkedShiftImporter(FromRattail, model.WorkedShiftImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class CustomerImporter(FromRattail, importing.model.CustomerImporter):

			




	
	
	
		
 
class CustomerImporter(FromRattail, model.CustomerImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class CustomerGroupImporter(FromRattail, importing.model.CustomerGroupImporter):

			




	
	
	
		
 
class CustomerGroupImporter(FromRattail, model.CustomerGroupImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class CustomerGroupAssignmentImporter(FromRattail, importing.model.CustomerGroupAssignmentImporter):

			




	
	
	
		
 
class CustomerGroupAssignmentImporter(FromRattail, model.CustomerGroupAssignmentImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class CustomerPersonImporter(FromRattail, importing.model.CustomerPersonImporter):

			




	
	
	
		
 
class CustomerPersonImporter(FromRattail, model.CustomerPersonImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class CustomerEmailAddressImporter(FromRattail, importing.model.CustomerEmailAddressImporter):

			




	
	
	
		
 
class CustomerEmailAddressImporter(FromRattail, model.CustomerEmailAddressImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class CustomerPhoneNumberImporter(FromRattail, importing.model.CustomerPhoneNumberImporter):

			




	
	
	
		
 
class CustomerPhoneNumberImporter(FromRattail, model.CustomerPhoneNumberImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class VendorImporter(FromRattail, importing.model.VendorImporter):

			




	
	
	
		
 
class VendorImporter(FromRattail, model.VendorImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class VendorEmailAddressImporter(FromRattail, importing.model.VendorEmailAddressImporter):

			




	
	
	
		
 
class VendorEmailAddressImporter(FromRattail, model.VendorEmailAddressImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class VendorPhoneNumberImporter(FromRattail, importing.model.VendorPhoneNumberImporter):

			




	
	
	
		
 
class VendorPhoneNumberImporter(FromRattail, model.VendorPhoneNumberImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class VendorContactImporter(FromRattail, importing.model.VendorContactImporter):

			




	
	
	
		
 
class VendorContactImporter(FromRattail, model.VendorContactImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class DepartmentImporter(FromRattail, importing.model.DepartmentImporter):

			




	
	
	
		
 
class DepartmentImporter(FromRattail, model.DepartmentImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class SubdepartmentImporter(FromRattail, importing.model.SubdepartmentImporter):

			




	
	
	
		
 
class SubdepartmentImporter(FromRattail, model.SubdepartmentImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class CategoryImporter(FromRattail, importing.model.CategoryImporter):

			




	
	
	
		
 
class CategoryImporter(FromRattail, model.CategoryImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class FamilyImporter(FromRattail, importing.model.FamilyImporter):

			




	
	
	
		
 
class FamilyImporter(FromRattail, model.FamilyImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class ReportCodeImporter(FromRattail, importing.model.ReportCodeImporter):

			




	
	
	
		
 
class ReportCodeImporter(FromRattail, model.ReportCodeImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class DepositLinkImporter(FromRattail, importing.model.DepositLinkImporter):

			




	
	
	
		
 
class DepositLinkImporter(FromRattail, model.DepositLinkImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class TaxImporter(FromRattail, importing.model.TaxImporter):

			




	
	
	
		
 
class TaxImporter(FromRattail, model.TaxImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class BrandImporter(FromRattail, importing.model.BrandImporter):

			




	
	
	
		
 
class BrandImporter(FromRattail, model.BrandImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class ProductImporter(FromRattail, importing.model.ProductImporter):

			




	
	
	
		
 
class ProductImporter(FromRattail, model.ProductImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class ProductCodeImporter(FromRattail, importing.model.ProductCodeImporter):

			




	
	
	
		
 
class ProductCodeImporter(FromRattail, model.ProductCodeImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class ProductCostImporter(FromRattail, importing.model.ProductCostImporter):

			




	
	
	
		
 
class ProductCostImporter(FromRattail, model.ProductCostImporter):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 
class ProductPriceImporter(FromRattail, importing.model.ProductPriceImporter):

			




	
	
	
		
 
class ProductPriceImporter(FromRattail, model.ProductPriceImporter):

			




	
	
	
		
 
    pass

			




        

    


    
    

    

        

                

                    rattail/tests/commands/test_importing.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -94,8 +94,11 @@ class TestImportSubcommandRun(ImporterTester, TestCase):

			




	
	
	
		
 

			




	
	
	
		
 
        kw = {

			




	
	
	
		
 
            'warnings': False,

			




	
	
	
		
 
            'create': None,

			




	
	
	
		
 
            'max_create': None,

			




	
	
	
		
 
            'update': None,

			




	
	
	
		
 
            'max_update': None,

			




	
	
	
		
 
            'delete': None,

			




	
	
	
		
 
            'max_delete': None,

			




	
	
	
		
 
            'max_total': None,

			




	
	
	
		
 
            'progress': None,

			




	
	
		
 
@@ -141,7 +144,7 @@ class TestImportSubcommandRun(ImporterTester, TestCase):

			




	
	
	
		
 
        local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data()

			




	
	
	
		
 
                self.import_data(delete=True)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus')

			




	
	
		
 
@@ -206,7 +209,7 @@ class TestImportSubcommandRun(ImporterTester, TestCase):

			




	
	
	
		
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(max_delete=1)

			




	
	
	
		
 
                self.import_data(delete=True, max_delete=1)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus1')

			




	
	
		
 
@@ -217,7 +220,7 @@ class TestImportSubcommandRun(ImporterTester, TestCase):

			




	
	
	
		
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(max_total=1)

			




	
	
	
		
 
                self.import_data(delete=True, max_total=1)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus1')

			




	
	
		
 
@@ -229,7 +232,7 @@ class TestImportSubcommandRun(ImporterTester, TestCase):

			




	
	
	
		
 
        local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(dry_run=True)

			




	
	
	
		
 
                self.import_data(delete=True, dry_run=True)

			




	
	
	
		
 
        # TODO: maybe need a way to confirm no changes actually made due to dry

			




	
	
	
		
 
        # run; currently results still reflect "proposed" changes.  this rather

			




	
	
	
		
 
        # bogus test is here just for coverage sake

			




        

    


    
    

    

        

                

                    rattail/tests/importing/test_handlers.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -2,8 +2,10 @@

			




	
	
	
		
 

			




	
	
	
		
 
from __future__ import unicode_literals, absolute_import

			




	
	
	
		
 

			




	
	
	
		
 
import datetime

			




	
	
	
		
 
import unittest

			




	
	
	
		
 

			




	
	
	
		
 
import pytz

			




	
	
	
		
 
from sqlalchemy import orm

			




	
	
	
		
 
from mock import patch, Mock

			




	
	
	
		
 
from fixture import TempIO

			




	
	
		
 
@@ -17,7 +19,7 @@ from rattail.tests.importing.test_importers import MockImporter

			




	
	
	
		
 
from rattail.tests.importing.test_postgresql import MockBulkImporter

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class TestImportHandlerBasics(unittest.TestCase):

			




	
	
	
		
 
class TestImportHandler(unittest.TestCase):

			




	
	
	
		
 

			




	
	
	
		
 
    def test_init(self):

			




	
	
	
		
 

			




	
	
		
 
@@ -27,6 +29,7 @@ class TestImportHandlerBasics(unittest.TestCase):

			




	
	
	
		
 
        self.assertEqual(handler.get_importers(), {})

			




	
	
	
		
 
        self.assertEqual(handler.get_importer_keys(), [])

			




	
	
	
		
 
        self.assertEqual(handler.get_default_keys(), [])

			




	
	
	
		
 
        self.assertFalse(handler.commit_host_partial)

			




	
	
	
		
 

			




	
	
	
		
 
        # with config

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
		
 
@@ -116,6 +119,14 @@ class TestImportHandlerBasics(unittest.TestCase):

			




	
	
	
		
 
                begin_host.assert_called_once_with()

			




	
	
	
		
 
                begin_local.assert_called_once_with()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_begin_host_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.begin_host_transaction()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_begin_local_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.begin_local_transaction()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_commit_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        with patch.object(handler, 'commit_host_transaction') as commit_host:

			




	
	
		
 
@@ -124,6 +135,14 @@ class TestImportHandlerBasics(unittest.TestCase):

			




	
	
	
		
 
                commit_host.assert_called_once_with()

			




	
	
	
		
 
                commit_local.assert_called_once_with()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_commit_host_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.commit_host_transaction()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_commit_local_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.commit_local_transaction()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_rollback_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        with patch.object(handler, 'rollback_host_transaction') as rollback_host:

			




	
	
		
 
@@ -132,6 +151,119 @@ class TestImportHandlerBasics(unittest.TestCase):

			




	
	
	
		
 
                rollback_host.assert_called_once_with()

			




	
	
	
		
 
                rollback_local.assert_called_once_with()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_rollback_host_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.rollback_host_transaction()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_rollback_local_transaction(self):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.rollback_local_transaction()

			




	
	
	
		
 

			




	
	
	
		
 
    def test_import_data(self):

			




	
	
	
		
 

			




	
	
	
		
 
        # normal

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        result = handler.import_data()

			




	
	
	
		
 
        self.assertEqual(result, {})

			




	
	
	
		
 

			




	
	
	
		
 
    def test_import_data_dry_run(self):

			




	
	
	
		
 

			




	
	
	
		
 
        # as init kwarg

			




	
	
	
		
 
        handler = handlers.ImportHandler(dry_run=True)

			




	
	
	
		
 
        with patch.object(handler, 'commit_transaction') as commit:

			




	
	
	
		
 
            with patch.object(handler, 'rollback_transaction') as rollback:

			




	
	
	
		
 
                handler.import_data()

			




	
	
	
		
 
                self.assertFalse(commit.called)

			




	
	
	
		
 
                rollback.assert_called_once_with()

			




	
	
	
		
 
        self.assertTrue(handler.dry_run)

			




	
	
	
		
 

			




	
	
	
		
 
        # as import kwarg

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        with patch.object(handler, 'commit_transaction') as commit:

			




	
	
	
		
 
            with patch.object(handler, 'rollback_transaction') as rollback:

			




	
	
	
		
 
                handler.import_data(dry_run=True)

			




	
	
	
		
 
                self.assertFalse(commit.called)

			




	
	
	
		
 
                rollback.assert_called_once_with()

			




	
	
	
		
 
        self.assertTrue(handler.dry_run)

			




	
	
	
		
 

			




	
	
	
		
 
    def test_import_data_invalid_model(self):

			




	
	
	
		
 
        importer = Mock()

			




	
	
	
		
 
        importer.import_data.return_value = [], [], []

			




	
	
	
		
 
        FooImporter = Mock(return_value=importer)

			




	
	
	
		
 

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.importers = {'Foo': FooImporter}

			




	
	
	
		
 

			




	
	
	
		
 
        handler.import_data('Foo')

			




	
	
	
		
 
        self.assertEqual(FooImporter.call_count, 1)

			




	
	
	
		
 
        importer.import_data.assert_called_once_with()

			




	
	
	
		
 

			




	
	
	
		
 
        FooImporter.reset_mock()

			




	
	
	
		
 
        importer.reset_mock()

			




	
	
	
		
 

			




	
	
	
		
 
        handler.import_data('Missing')

			




	
	
	
		
 
        self.assertFalse(FooImporter.called)

			




	
	
	
		
 
        self.assertFalse(importer.called)

			




	
	
	
		
 

			




	
	
	
		
 
    def test_import_data_with_changes(self):

			




	
	
	
		
 
        importer = Mock()

			




	
	
	
		
 
        FooImporter = Mock(return_value=importer)

			




	
	
	
		
 

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.importers = {'Foo': FooImporter}

			




	
	
	
		
 

			




	
	
	
		
 
        importer.import_data.return_value = [], [], []

			




	
	
	
		
 
        with patch.object(handler, 'process_changes') as process:

			




	
	
	
		
 
            handler.import_data('Foo')

			




	
	
	
		
 
            self.assertFalse(process.called)

			




	
	
	
		
 

			




	
	
	
		
 
        importer.import_data.return_value = [1], [2], [3]

			




	
	
	
		
 
        with patch.object(handler, 'process_changes') as process:

			




	
	
	
		
 
            handler.import_data('Foo')

			




	
	
	
		
 
            process.assert_called_once_with({'Foo': ([1], [2], [3])})

			




	
	
	
		
 

			




	
	
	
		
 
    def test_import_data_commit_host_partial(self):

			




	
	
	
		
 
        importer = Mock()

			




	
	
	
		
 
        importer.import_data.side_effect = ValueError

			




	
	
	
		
 
        FooImporter = Mock(return_value=importer)

			




	
	
	
		
 

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.importers = {'Foo': FooImporter}

			




	
	
	
		
 

			




	
	
	
		
 
        handler.commit_host_partial = False

			




	
	
	
		
 
        with patch.object(handler, 'commit_host_transaction') as commit:

			




	
	
	
		
 
            self.assertRaises(ValueError, handler.import_data, 'Foo')

			




	
	
	
		
 
            self.assertFalse(commit.called)

			




	
	
	
		
 

			




	
	
	
		
 
        handler.commit_host_partial = True

			




	
	
	
		
 
        with patch.object(handler, 'commit_host_transaction') as commit:

			




	
	
	
		
 
            self.assertRaises(ValueError, handler.import_data, 'Foo')

			




	
	
	
		
 
            commit.assert_called_once_with()

			




	
	
	
		
 

			




	
	
	
		
 
    @patch('rattail.importing.handlers.send_email')

			




	
	
	
		
 
    def test_process_changes_sends_email(self, send_email):

			




	
	
	
		
 
        handler = handlers.ImportHandler()

			




	
	
	
		
 
        handler.import_began = pytz.utc.localize(datetime.datetime.utcnow())

			




	
	
	
		
 
        changes = [], [], []

			




	
	
	
		
 

			




	
	
	
		
 
        # warnings disabled

			




	
	
	
		
 
        handler.warnings = False

			




	
	
	
		
 
        handler.process_changes(changes)

			




	
	
	
		
 
        self.assertFalse(send_email.called)

			




	
	
	
		
 

			




	
	
	
		
 
        # warnings enabled

			




	
	
	
		
 
        handler.warnings = True

			




	
	
	
		
 
        handler.process_changes(changes)

			




	
	
	
		
 
        self.assertEqual(send_email.call_count, 1)

			




	
	
	
		
 

			




	
	
	
		
 
        send_email.reset_mock()

			




	
	
	
		
 

			




	
	
	
		
 
        # warnings enabled, with command (just for coverage..)

			




	
	
	
		
 
        handler.warnings = True

			




	
	
	
		
 
        handler.command = Mock(name='import-testing', parent=Mock(name='rattail'))

			




	
	
	
		
 
        handler.process_changes(changes)

			




	
	
	
		
 
        self.assertEqual(send_email.call_count, 1)

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
######################################################################

			




	
	
	
		
 
# fake import handler, tested mostly for basic coverage

			




	
	
		
 
@@ -203,7 +335,7 @@ class TestImportHandlerImportData(ImporterTester, unittest.TestCase):

			




	
	
	
		
 
        local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data()

			




	
	
	
		
 
                self.import_data(delete=True)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus')

			




	
	
		
 
@@ -268,7 +400,7 @@ class TestImportHandlerImportData(ImporterTester, unittest.TestCase):

			




	
	
	
		
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(max_delete=1)

			




	
	
	
		
 
                self.import_data(delete=True, max_delete=1)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus1')

			




	
	
		
 
@@ -279,7 +411,7 @@ class TestImportHandlerImportData(ImporterTester, unittest.TestCase):

			




	
	
	
		
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(max_total=1)

			




	
	
	
		
 
                self.import_data(delete=True, max_total=1)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus1')

			




	
	
		
 
@@ -291,7 +423,7 @@ class TestImportHandlerImportData(ImporterTester, unittest.TestCase):

			




	
	
	
		
 
        local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(dry_run=True)

			




	
	
	
		
 
                self.import_data(delete=True, dry_run=True)

			




	
	
	
		
 
        # TODO: maybe need a way to confirm no changes actually made due to dry

			




	
	
	
		
 
        # run; currently results still reflect "proposed" changes.  this rather

			




	
	
	
		
 
        # bogus test is here just for coverage sake

			




	
	
		
 
@@ -308,7 +440,7 @@ class TestImportHandlerImportData(ImporterTester, unittest.TestCase):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                with patch('rattail.importing.handlers.send_email') as send_email:

			




	
	
	
		
 
                    self.assertEqual(send_email.call_count, 0)

			




	
	
	
		
 
                    self.import_data(warnings=True, dry_run=True)

			




	
	
	
		
 
                    self.import_data(delete=True, warnings=True, dry_run=True)

			




	
	
	
		
 
                    self.assertEqual(send_email.call_count, 1)

			




	
	
	
		
 
        # second time is just for more coverage...

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
		
 
@@ -316,7 +448,7 @@ class TestImportHandlerImportData(ImporterTester, unittest.TestCase):

			




	
	
	
		
 
                with patch('rattail.importing.handlers.send_email') as send_email:

			




	
	
	
		
 
                    self.handler.command = Mock()

			




	
	
	
		
 
                    self.assertEqual(send_email.call_count, 0)

			




	
	
	
		
 
                    self.import_data(warnings=True)

			




	
	
	
		
 
                    self.import_data(delete=True, warnings=True)

			




	
	
	
		
 
                    self.assertEqual(send_email.call_count, 1)

			




	
	
	
		
 
        # TODO: maybe need a way to confirm no changes actually made due to dry

			




	
	
	
		
 
        # run; currently results still reflect "proposed" changes.  this rather

			




        

    


    
    

    

        

                

                    rattail/tests/importing/test_importers.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -36,6 +36,23 @@ class TestImporter(TestCase):

			




	
	
	
		
 
        importer = importers.Importer(extra_bit=extra_bit)

			




	
	
	
		
 
        self.assertIs(importer.extra_bit, extra_bit)

			




	
	
	
		
 

			




	
	
	
		
 
    def test_delete_flag(self):

			




	
	
	
		
 
        # disabled by default

			




	
	
	
		
 
        importer = importers.Importer()

			




	
	
	
		
 
        self.assertTrue(importer.allow_delete)

			




	
	
	
		
 
        self.assertFalse(importer.delete)

			




	
	
	
		
 
        importer.import_data(host_data=[])

			




	
	
	
		
 
        self.assertFalse(importer.delete)

			




	
	
	
		
 

			




	
	
	
		
 
        # but can be enabled

			




	
	
	
		
 
        importer = importers.Importer(delete=True)

			




	
	
	
		
 
        self.assertTrue(importer.allow_delete)

			




	
	
	
		
 
        self.assertTrue(importer.delete)

			




	
	
	
		
 
        importer = importers.Importer()

			




	
	
	
		
 
        self.assertFalse(importer.delete)

			




	
	
	
		
 
        importer.import_data(host_data=[], delete=True)

			




	
	
	
		
 
        self.assertTrue(importer.delete)

			




	
	
	
		
 

			




	
	
	
		
 
    def test_get_host_objects(self):

			




	
	
	
		
 
        importer = importers.Importer()

			




	
	
	
		
 
        objects = importer.get_host_objects()

			




	
	
		
 
@@ -217,7 +234,7 @@ class TestMockImporter(ImporterTester, TestCase):

			




	
	
	
		
 
        local['bogus'] = {'upc': '00000000000000', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data()

			




	
	
	
		
 
                self.import_data(delete=True)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus')

			




	
	
		
 
@@ -282,7 +299,7 @@ class TestMockImporter(ImporterTester, TestCase):

			




	
	
	
		
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(max_delete=1)

			




	
	
	
		
 
                self.import_data(delete=True, max_delete=1)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus1')

			




	
	
		
 
@@ -293,7 +310,21 @@ class TestMockImporter(ImporterTester, TestCase):

			




	
	
	
		
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(max_total=1)

			




	
	
	
		
 
                self.import_data(delete=True, max_total=1)

			




	
	
	
		
 
        self.assert_import_created()

			




	
	
	
		
 
        self.assert_import_updated()

			




	
	
	
		
 
        self.assert_import_deleted('bogus1')

			




	
	
	
		
 

			




	
	
	
		
 
    def test_max_total_delete_with_changes(self):

			




	
	
	
		
 
        local = self.copy_data()

			




	
	
	
		
 
        del local['16oz']

			




	
	
	
		
 
        local['32oz']['description'] = "wrong"

			




	
	
	
		
 
        local['1gal']['description'] = "wrong"

			




	
	
	
		
 
        local['bogus1'] = {'upc': '00000000000001', 'description': "Delete Me"}

			




	
	
	
		
 
        local['bogus2'] = {'upc': '00000000000002', 'description': "Delete Me"}

			




	
	
	
		
 
        with self.host_data(self.sample_data):

			




	
	
	
		
 
            with self.local_data(local):

			




	
	
	
		
 
                self.import_data(delete=True, max_total=3)

			




	
	
	
		
 
        self.assert_import_created('16oz')

			




	
	
	
		
 
        self.assert_import_updated('32oz', '1gal')

			




	
	
	
		
 
        self.assert_import_deleted()

			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







  


    




    








    
        Server instance: tuna-2793187
    
    
        This site is powered by
            Kallithea 0.7.0,
        which is
        © 2010–2021 by various authors & licensed under GPLv3.
            – Support