From 15003f47d9343c2218043184b37d70b354a6fda1 2015-08-21 22:14:20 From: Lance Edgar Date: 2015-08-21 22:14:20 Subject: [PATCH] Add `ImportHandler` class, update `ImportSubcommand` to use it etc. --- diff --git a/rattail/commands.py b/rattail/commands.py index 1ab1d493e3b31cb800d91d30373da2cd81a2e548..dbfaee508e184eb1f56c19e358b779a35d63626c 100644 --- a/rattail/commands.py +++ b/rattail/commands.py @@ -39,9 +39,10 @@ import logging from getpass import getpass from ._version import __version__ -from .util import load_entry_points +from rattail.util import load_entry_points, load_object from .console import Progress from rattail.config import make_config +from rattail.mail import send_email log = logging.getLogger(__name__) @@ -603,6 +604,7 @@ class ImportSubcommand(Subcommand): supports_versioning = True def add_parser_args(self, parser): + handler = self.get_handler() if self.supports_versioning: parser.add_argument('--no-versioning', action='store_true', help="Disables versioning during the import. This is " @@ -622,8 +624,9 @@ class ImportSubcommand(Subcommand): help="Go through the motions and allow logging to occur, " "but do not actually commit the transaction at the end.") parser.add_argument('models', nargs='*', metavar='MODEL', - help="One or more models to import. If not specified, " - "then all supported models will be imported.") + help="Which models to import. If none are specified, all models will " + "be imported. Or, specify only those you wish to import. Supported " + "models are: {0}".format(', '.join(handler.get_importer_keys()))) def run(self, args): log.info("begin {0} for data model(s): {1}".format( @@ -648,6 +651,21 @@ class ImportSubcommand(Subcommand): log.info("transaction was committed") session.close() + def get_handler_factory(self): + """ + This method must return a factory, which will in turn generate a + handler instance to be used by the command. Note that you *must* + override this method. + """ + raise NotImplementedError + + def get_handler(self, **kwargs): + """ + Returns a handler instance to be used by the command. + """ + factory = self.get_handler_factory() + return factory(getattr(self, 'config', None), **kwargs) + @property def continuum_user(self): """ @@ -656,8 +674,30 @@ class ImportSubcommand(Subcommand): def import_data(self, args, session): """ - Custom logic for importing data, to be implemented by subclass. + Perform a data import, with the given arguments and database session. + """ + handler = self.get_handler(session=session) + models = args.models or handler.get_importer_keys() + updates = handler.import_data(models, max_updates=args.max_updates, + progress=self.progress) + if args.warnings and updates: + key = '{0}_{1}_updates'.format(self.parent.name, self.name) + key = key.replace('-', '_') + send_email(self.config, key, fallback_key='rattail_import_updates', data={ + 'command': '{0} {1}'.format(self.parent.name, self.name), + 'models': models, + 'dry_run': args.dry_run, + 'updates': updates, + 'render_record': self.get_record_renderer()}) + + def get_record_renderer(self): + """ + Get the record renderer for email notifications. Note that config may + override the default. """ + spec = self.config.get('{0}.{1}'.format(self.parent.name, self.name), 'record_renderer', + default='rattail.db.importing:RecordRenderer') + return load_object(spec)(self.config) class ImportCSV(ImportSubcommand): diff --git a/rattail/db/importing/__init__.py b/rattail/db/importing/__init__.py index 42c7abb952d2c9f3da2c189907ec138b9e12a99d..9b50b7f13b020cc17ec581aea4296017de4397d7 100644 --- a/rattail/db/importing/__init__.py +++ b/rattail/db/importing/__init__.py @@ -24,6 +24,7 @@ Data Importing """ -from .core import Importer, make_importer +from .core import Importer, make_importer, RecordRenderer from . import models from .providers import DataProvider, QueryProvider +from .handlers import ImportHandler diff --git a/rattail/db/importing/core.py b/rattail/db/importing/core.py index bb188b0266af5e08c48d9383448a87b860f5e9c9..815a6edbc6831a1d058261bdcb687b3cb50d814f 100644 --- a/rattail/db/importing/core.py +++ b/rattail/db/importing/core.py @@ -130,6 +130,9 @@ class Importer(Object): if progress: prog = progress("Importing {0} data".format(self.model_name), count) + created = [] + updated = [] + affected = 0 keys_seen = set() for i, src_data in enumerate(records, 1): @@ -147,6 +150,7 @@ class Importer(Object): if self.data_differs(inst_data, src_data): instance = self.get_instance(src_data) self.update_instance(instance, src_data, inst_data) + updated.append(instance) dirty = True else: instance = self.new_instance(src_data) @@ -155,6 +159,7 @@ class Importer(Object): self.session.add(instance) self.session.flush() log.debug("created new {0} {1}: {2}".format(self.model_name, instance.uuid, instance)) + created.append(instance) dirty = True if dirty: @@ -169,7 +174,7 @@ class Importer(Object): if prog: prog.destroy() - return affected + return created, updated def setup(self, progress): """ @@ -337,3 +342,60 @@ class Importer(Object): if field in data: if not inst_data or inst_data[field] != data[field]: setattr(instance, field, data[field]) + + +class RecordRenderer(object): + """ + Record renderer for email notifications sent from data import jobs. + """ + + def __init__(self, config): + self.config = config + + def __call__(self, record): + return self.render(record) + + def render(self, record): + """ + Render the given record. Default is to attempt. + """ + key = record.__class__.__name__.lower() + renderer = getattr(self, 'render_{0}'.format(key), None) + if renderer: + return renderer(record) + + label = self.get_label(record) + url = self.get_url(record) + if url: + return '{1}'.format(url, label) + return label + + def get_label(self, record): + key = record.__class__.__name__.lower() + label = getattr(self, 'label_{0}'.format(key), self.label) + return label(record) + + def label(self, record): + return unicode(record) + + def get_url(self, record): + """ + Fetch / generate a URL for the given data record. You should *not* + override this method, but do :meth:`url()` instead. + """ + key = record.__class__.__name__.lower() + url = getattr(self, 'url_{0}'.format(key), self.url) + return url(record) + + def url(self, record): + """ + Fetch / generate a URL for the given data record. + """ + url = self.config.get('tailbone', 'url') + if url: + url = url.rstrip('/') + name = '{0}s'.format(record.__class__.__name__.lower()) + if name == 'persons': # FIXME, obviously this is a hack + name = 'people' + url = '{0}/{1}/{{uuid}}'.format(url, name) + return url.format(uuid=record.uuid) diff --git a/rattail/db/importing/handlers.py b/rattail/db/importing/handlers.py new file mode 100644 index 0000000000000000000000000000000000000000..6177175d2e64ade5c624382a8809d2fdbf51b957 --- /dev/null +++ b/rattail/db/importing/handlers.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2015 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Rattail. If not, see . +# +################################################################################ +""" +Import Handlers +""" + +from __future__ import unicode_literals + +import logging + +from rattail.util import OrderedDict + + +log = logging.getLogger(__name__) + + +class ImportHandler(object): + """ + Base class for all import handlers. + """ + + def __init__(self, config=None, session=None): + self.config = config + self.session = session + self.importers = self.get_importers() + + def get_importers(self): + """ + Returns a dict of all available importers, where the values are + importer factories. All subclasses will want to override this. Note + that if you return an ``OrderedDict`` instance, you can affect the + ordering of keys in the command line help system, etc. + """ + return {} + + def get_importer_keys(self): + """ + Returns a list of keys corresponding to the available importers. + """ + return list(self.importers.iterkeys()) + + def get_importer(self, key): + """ + Returns an importer instance corresponding to the given key. + """ + return self.importers[key](self.config, self.session, + **self.get_importer_kwargs(key)) + + def get_importer_kwargs(self, key): + """ + Return a dict of kwargs to be used when construcing an importer with + the given key. + """ + return {} + + def import_data(self, keys, max_updates=None, progress=None): + """ + Import all data for the given importer keys. + """ + self.before_import() + updates = OrderedDict() + + for key in keys: + provider = self.get_importer(key) + if not provider: + log.warning("unknown importer; skipping: {0}".format(repr(key))) + continue + + data = provider.get_data(progress=progress) + created, updated = provider.importer.import_data( + data, provider.supported_fields, provider.key, + max_updates=max_updates, progress=progress) + + if hasattr(provider, 'process_deletions'): + deleted = provider.process_deletions(data, progress=progress) + else: + deleted = 0 + + log.info("added {0}, updated {1}, deleted {2} {3} records".format( + len(created), len(updated), deleted, key)) + if created or updated or deleted: + updates[key] = created, updated, deleted + + self.after_import() + return updates + + def before_import(self): + return + + def after_import(self): + return diff --git a/rattail/templates/mail/rattail_import_updates.html.mako b/rattail/templates/mail/rattail_import_updates.html.mako new file mode 100644 index 0000000000000000000000000000000000000000..9f3f5704a1af7d172230ea67808675d58168aaec --- /dev/null +++ b/rattail/templates/mail/rattail_import_updates.html.mako @@ -0,0 +1,52 @@ + + +

Data Import Warnings (${command})

+ % if dry_run: +

+ NOTE:  This was a dry run only; no data was harmed + in the making of this email. +

+ % endif +

+ Generally the periodic data import is expected to be a precaution only, in order + to detect and fix Rattail data which falls out of sync from the data authority, + e.g. your POS.  It is normally expected that proper real-time operation + should be enough to keep things in sync; therefore any actual changes + which occur as part of the import process are considered "warnings". +

+

+ The following is a list of changes which occurred during the latest + import run.  Please investigate at your convenience. +

+ + % for model, (created, updated, deleted) in updates.iteritems(): +

${model}

+ % for label, records in (('created', created), ('updated', updated)): + % if records: + % if len(records) == 1: +

1 record was ${label}:

+ % else: +

${len(records)} records were ${label}:

+ % endif + + % endif + % endfor + % if deleted == 1: +

1 record was deleted

+ % elif deleted: +

${deleted} records were deleted

+ % endif + % endfor + +