Changeset - 15003f47d934
[Not reviewed]
0 3 2
Lance Edgar - 9 years ago 2015-08-21 22:14:20
ledgar@sacfoodcoop.com
Add `ImportHandler` class, update `ImportSubcommand` to use it etc.
5 files changed with 272 insertions and 6 deletions:
0 comments (0 inline, 0 general)
rattail/commands.py
Show inline comments
 
@@ -36,15 +36,16 @@ import socket
 
import shutil
 
import warnings
 
import logging
 
from getpass import getpass
 

	
 
from ._version import __version__
 
from .util import load_entry_points
 
from rattail.util import load_entry_points, load_object
 
from .console import Progress
 
from rattail.config import make_config
 
from rattail.mail import send_email
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class ArgumentParser(argparse.ArgumentParser):
 
@@ -600,12 +601,13 @@ class ImportSubcommand(Subcommand):
 
    """
 
    Base class for subcommands which use the data importing system.
 
    """
 
    supports_versioning = True
 

	
 
    def add_parser_args(self, parser):
 
        handler = self.get_handler()
 
        if self.supports_versioning:
 
            parser.add_argument('--no-versioning', action='store_true',
 
                                help="Disables versioning during the import.  This is "
 
                                "intended to be useful e.g. during initial import, where "
 
                                "the process can be quite slow even without the overhead "
 
                                "of versioning.")
 
@@ -619,14 +621,15 @@ class ImportSubcommand(Subcommand):
 
                            "updates which have completed will be committed unless a dry run "
 
                            "is in effect.")
 
        parser.add_argument('--dry-run', action='store_true',
 
                            help="Go through the motions and allow logging to occur, "
 
                            "but do not actually commit the transaction at the end.")
 
        parser.add_argument('models', nargs='*', metavar='MODEL',
 
                            help="One or more models to import.  If not specified, "
 
                            "then all supported models will be imported.")
 
                            help="Which models to import.  If none are specified, all models will "
 
                            "be imported.  Or, specify only those you wish to import.  Supported "
 
                            "models are: {0}".format(', '.join(handler.get_importer_keys())))
 

	
 
    def run(self, args):
 
        log.info("begin {0} for data model(s): {1}".format(
 
                self.name, ', '.join(args.models or ["ALL"])))
 

	
 
        Session = self.parent.db_session_factory
 
@@ -645,22 +648,59 @@ class ImportSubcommand(Subcommand):
 
            log.info("dry run, so transaction was rolled back")
 
        else:
 
            session.commit()
 
            log.info("transaction was committed")
 
        session.close()
 

	
 
    def get_handler_factory(self):
 
        """
 
        This method must return a factory, which will in turn generate a
 
        handler instance to be used by the command.  Note that you *must*
 
        override this method.
 
        """
 
        raise NotImplementedError
 

	
 
    def get_handler(self, **kwargs):
 
        """
 
        Returns a handler instance to be used by the command.
 
        """
 
        factory = self.get_handler_factory()
 
        return factory(getattr(self, 'config', None), **kwargs)
 

	
 
    @property
 
    def continuum_user(self):
 
        """
 
        Info needed to assign the Continuum user for the database session.
 
        """
 

	
 
    def import_data(self, args, session):
 
        """
 
        Custom logic for importing data, to be implemented by subclass.
 
        Perform a data import, with the given arguments and database session.
 
        """
 
        handler = self.get_handler(session=session)
 
        models = args.models or handler.get_importer_keys()
 
        updates = handler.import_data(models, max_updates=args.max_updates,
 
                                      progress=self.progress)
 
        if args.warnings and updates:
 
            key = '{0}_{1}_updates'.format(self.parent.name, self.name)
 
            key = key.replace('-', '_')
 
            send_email(self.config, key, fallback_key='rattail_import_updates', data={
 
                'command': '{0} {1}'.format(self.parent.name, self.name),
 
                'models': models,
 
                'dry_run': args.dry_run,
 
                'updates': updates,
 
                'render_record': self.get_record_renderer()})
 

	
 
    def get_record_renderer(self):
 
        """
 
        Get the record renderer for email notifications.  Note that config may
 
        override the default.
 
        """
 
        spec = self.config.get('{0}.{1}'.format(self.parent.name, self.name), 'record_renderer',
 
                               default='rattail.db.importing:RecordRenderer')
 
        return load_object(spec)(self.config)
 

	
 

	
 
class ImportCSV(ImportSubcommand):
 
    """
 
    Import data from a CSV file.
 
    """
rattail/db/importing/__init__.py
Show inline comments
 
@@ -21,9 +21,10 @@
 
#
 
################################################################################
 
"""
 
Data Importing
 
"""
 

	
 
from .core import Importer, make_importer
 
from .core import Importer, make_importer, RecordRenderer
 
from . import models
 
from .providers import DataProvider, QueryProvider
 
from .handlers import ImportHandler
rattail/db/importing/core.py
Show inline comments
 
@@ -127,12 +127,15 @@ class Importer(Object):
 
            records.append(normalized[key])
 

	
 
        prog = None
 
        if progress:
 
            prog = progress("Importing {0} data".format(self.model_name), count)
 

	
 
        created = []
 
        updated = []
 

	
 
        affected = 0
 
        keys_seen = set()
 
        for i, src_data in enumerate(records, 1):
 
            key = self.get_key(src_data)
 
            if key in keys_seen:
 
                log.warning("duplicate records from {0}:{1} for key: {2}".format(
 
@@ -144,20 +147,22 @@ class Importer(Object):
 
            dirty = False
 
            inst_data = self.get_instance_data(src_data)
 
            if inst_data:
 
                if self.data_differs(inst_data, src_data):
 
                    instance = self.get_instance(src_data)
 
                    self.update_instance(instance, src_data, inst_data)
 
                    updated.append(instance)
 
                    dirty = True
 
            else:
 
                instance = self.new_instance(src_data)
 
                assert instance, "Failed to create new model instance for data: {0}".format(repr(src_data))
 
                self.update_instance(instance, src_data)
 
                self.session.add(instance)
 
                self.session.flush()
 
                log.debug("created new {0} {1}: {2}".format(self.model_name, instance.uuid, instance))
 
                created.append(instance)
 
                dirty = True
 

	
 
            if dirty:
 
                self.session.flush()
 
                affected += 1
 
                if max_updates and affected >= max_updates:
 
@@ -166,13 +171,13 @@ class Importer(Object):
 

	
 
            if prog:
 
                prog.update(i)
 
        if prog:
 
            prog.destroy()
 

	
 
        return affected
 
        return created, updated
 

	
 
    def setup(self, progress):
 
        """
 
        Perform any setup necessary, e.g. cache lookups for existing data.
 
        """
 

	
 
@@ -334,6 +339,63 @@ class Importer(Object):
 
        Update the given model instance with the given data.
 
        """
 
        for field in self.simple_fields:
 
            if field in data:
 
                if not inst_data or inst_data[field] != data[field]:
 
                    setattr(instance, field, data[field])
 

	
 

	
 
class RecordRenderer(object):
 
    """
 
    Record renderer for email notifications sent from data import jobs.
 
    """
 

	
 
    def __init__(self, config):
 
        self.config = config
 

	
 
    def __call__(self, record):
 
        return self.render(record)
 

	
 
    def render(self, record):
 
        """
 
        Render the given record.  Default is to attempt.
 
        """
 
        key = record.__class__.__name__.lower()
 
        renderer = getattr(self, 'render_{0}'.format(key), None)
 
        if renderer:
 
            return renderer(record)
 

	
 
        label = self.get_label(record)
 
        url = self.get_url(record)
 
        if url:
 
            return '<a href="{0}">{1}</a>'.format(url, label)
 
        return label
 

	
 
    def get_label(self, record):
 
        key = record.__class__.__name__.lower()
 
        label = getattr(self, 'label_{0}'.format(key), self.label)
 
        return label(record)
 

	
 
    def label(self, record):
 
        return unicode(record)
 

	
 
    def get_url(self, record):
 
        """
 
        Fetch / generate a URL for the given data record.  You should *not*
 
        override this method, but do :meth:`url()` instead.
 
        """
 
        key = record.__class__.__name__.lower()
 
        url = getattr(self, 'url_{0}'.format(key), self.url)
 
        return url(record)
 

	
 
    def url(self, record):
 
        """
 
        Fetch / generate a URL for the given data record.
 
        """
 
        url = self.config.get('tailbone', 'url')
 
        if url:
 
            url = url.rstrip('/')
 
            name = '{0}s'.format(record.__class__.__name__.lower())
 
            if name == 'persons': # FIXME, obviously this is a hack
 
                name = 'people'
 
            url = '{0}/{1}/{{uuid}}'.format(url, name)
 
            return url.format(uuid=record.uuid)
rattail/db/importing/handlers.py
Show inline comments
 
new file 100644
 
# -*- coding: utf-8 -*-
 
################################################################################
 
#
 
#  Rattail -- Retail Software Framework
 
#  Copyright © 2010-2015 Lance Edgar
 
#
 
#  This file is part of Rattail.
 
#
 
#  Rattail is free software: you can redistribute it and/or modify it under the
 
#  terms of the GNU Affero General Public License as published by the Free
 
#  Software Foundation, either version 3 of the License, or (at your option)
 
#  any later version.
 
#
 
#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 
#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 
#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
 
#  more details.
 
#
 
#  You should have received a copy of the GNU Affero General Public License
 
#  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.
 
#
 
################################################################################
 
"""
 
Import Handlers
 
"""
 

	
 
from __future__ import unicode_literals
 

	
 
import logging
 

	
 
from rattail.util import OrderedDict
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class ImportHandler(object):
 
    """
 
    Base class for all import handlers.
 
    """
 

	
 
    def __init__(self, config=None, session=None):
 
        self.config = config
 
        self.session = session
 
        self.importers = self.get_importers()
 

	
 
    def get_importers(self):
 
        """
 
        Returns a dict of all available importers, where the values are
 
        importer factories.  All subclasses will want to override this.  Note
 
        that if you return an ``OrderedDict`` instance, you can affect the
 
        ordering of keys in the command line help system, etc.
 
        """
 
        return {}
 

	
 
    def get_importer_keys(self):
 
        """
 
        Returns a list of keys corresponding to the available importers.
 
        """
 
        return list(self.importers.iterkeys())
 

	
 
    def get_importer(self, key):
 
        """
 
        Returns an importer instance corresponding to the given key.
 
        """
 
        return self.importers[key](self.config, self.session,
 
                                   **self.get_importer_kwargs(key))
 

	
 
    def get_importer_kwargs(self, key):
 
        """
 
        Return a dict of kwargs to be used when construcing an importer with
 
        the given key.
 
        """
 
        return {}
 

	
 
    def import_data(self, keys, max_updates=None, progress=None):
 
        """
 
        Import all data for the given importer keys.
 
        """
 
        self.before_import()
 
        updates = OrderedDict()
 

	
 
        for key in keys:
 
            provider = self.get_importer(key)
 
            if not provider:
 
                log.warning("unknown importer; skipping: {0}".format(repr(key)))
 
                continue
 

	
 
            data = provider.get_data(progress=progress)
 
            created, updated = provider.importer.import_data(
 
                data, provider.supported_fields, provider.key,
 
                max_updates=max_updates, progress=progress)
 

	
 
            if hasattr(provider, 'process_deletions'):
 
                deleted = provider.process_deletions(data, progress=progress)
 
            else:
 
                deleted = 0
 

	
 
            log.info("added {0}, updated {1}, deleted {2} {3} records".format(
 
                len(created), len(updated), deleted, key))
 
            if created or updated or deleted:
 
                updates[key] = created, updated, deleted
 

	
 
        self.after_import()
 
        return updates
 

	
 
    def before_import(self):
 
        return 
 

	
 
    def after_import(self):
 
        return 
rattail/templates/mail/rattail_import_updates.html.mako
Show inline comments
 
new file 100644
 
<html>
 
  <body>
 
    <h3>Data Import Warnings (<code>${command}</code>)</h3>
 
    % if dry_run:
 
        <p>
 
          <em><strong>NOTE:</strong>&nbsp; This was a dry run only; no data was harmed
 
          in the making of this email.</em>
 
        </p>
 
    % endif
 
    <p>
 
      Generally the periodic data import is expected to be a precaution only, in order
 
      to detect and fix Rattail data which falls out of sync from the data authority,
 
      e.g. your POS.&nbsp; It is normally expected that proper real-time operation
 
      <em>should</em> be enough to keep things in sync; therefore any actual changes
 
      which occur as part of the import process are considered "warnings".
 
    </p>
 
    <p>
 
      The following is a list of changes which occurred during the latest
 
      import run.&nbsp; Please investigate at your convenience.
 
    </p>
 
    <ul>
 
      % for model, (created, updated, deleted) in updates.iteritems():
 
          <li>
 
            <a href="#${model}">${model}</a>
 
            - ${len(created)} created, ${len(updated)} updated, ${deleted} deleted
 
          </li>
 
      % endfor
 
    </ul>
 
    % for model, (created, updated, deleted) in updates.iteritems():
 
        <h4><a name="${model}">${model}</a></h4>
 
        % for label, records in (('created', created), ('updated', updated)):
 
            % if records:
 
                % if len(records) == 1:
 
                    <p>1 record was <strong>${label}:</strong></p>
 
                % else:
 
                    <p>${len(records)} records were <strong>${label}:</strong></p>
 
                % endif
 
                <ul>
 
                  % for record in records:
 
                      <li>${render_record(record)}</li>
 
                  % endfor
 
                </ul>
 
            % endif
 
        % endfor
 
        % if deleted == 1:
 
            <p>1 record was <strong>deleted</strong></p>
 
        % elif deleted:
 
            <p>${deleted} records were <strong>deleted</strong></p>
 
        % endif
 
    % endfor
 
  </body>
 
</html>
0 comments (0 inline, 0 general)