diff --git a/rattail/commands.py b/rattail/commands.py
index 1ab1d493e3b31cb800d91d30373da2cd81a2e548..dbfaee508e184eb1f56c19e358b779a35d63626c 100644
--- a/rattail/commands.py
+++ b/rattail/commands.py
@@ -39,9 +39,10 @@ import logging
from getpass import getpass
from ._version import __version__
-from .util import load_entry_points
+from rattail.util import load_entry_points, load_object
from .console import Progress
from rattail.config import make_config
+from rattail.mail import send_email
log = logging.getLogger(__name__)
@@ -603,6 +604,7 @@ class ImportSubcommand(Subcommand):
supports_versioning = True
def add_parser_args(self, parser):
+ handler = self.get_handler()
if self.supports_versioning:
parser.add_argument('--no-versioning', action='store_true',
help="Disables versioning during the import. This is "
@@ -622,8 +624,9 @@ class ImportSubcommand(Subcommand):
help="Go through the motions and allow logging to occur, "
"but do not actually commit the transaction at the end.")
parser.add_argument('models', nargs='*', metavar='MODEL',
- help="One or more models to import. If not specified, "
- "then all supported models will be imported.")
+ help="Which models to import. If none are specified, all models will "
+ "be imported. Or, specify only those you wish to import. Supported "
+ "models are: {0}".format(', '.join(handler.get_importer_keys())))
def run(self, args):
log.info("begin {0} for data model(s): {1}".format(
@@ -648,6 +651,21 @@ class ImportSubcommand(Subcommand):
log.info("transaction was committed")
session.close()
+ def get_handler_factory(self):
+ """
+ This method must return a factory, which will in turn generate a
+ handler instance to be used by the command. Note that you *must*
+ override this method.
+ """
+ raise NotImplementedError
+
+ def get_handler(self, **kwargs):
+ """
+ Returns a handler instance to be used by the command.
+ """
+ factory = self.get_handler_factory()
+ return factory(getattr(self, 'config', None), **kwargs)
+
@property
def continuum_user(self):
"""
@@ -656,8 +674,30 @@ class ImportSubcommand(Subcommand):
def import_data(self, args, session):
"""
- Custom logic for importing data, to be implemented by subclass.
+ Perform a data import, with the given arguments and database session.
+ """
+ handler = self.get_handler(session=session)
+ models = args.models or handler.get_importer_keys()
+ updates = handler.import_data(models, max_updates=args.max_updates,
+ progress=self.progress)
+ if args.warnings and updates:
+ key = '{0}_{1}_updates'.format(self.parent.name, self.name)
+ key = key.replace('-', '_')
+ send_email(self.config, key, fallback_key='rattail_import_updates', data={
+ 'command': '{0} {1}'.format(self.parent.name, self.name),
+ 'models': models,
+ 'dry_run': args.dry_run,
+ 'updates': updates,
+ 'render_record': self.get_record_renderer()})
+
+ def get_record_renderer(self):
+ """
+ Get the record renderer for email notifications. Note that config may
+ override the default.
"""
+ spec = self.config.get('{0}.{1}'.format(self.parent.name, self.name), 'record_renderer',
+ default='rattail.db.importing:RecordRenderer')
+ return load_object(spec)(self.config)
class ImportCSV(ImportSubcommand):
diff --git a/rattail/db/importing/__init__.py b/rattail/db/importing/__init__.py
index 42c7abb952d2c9f3da2c189907ec138b9e12a99d..9b50b7f13b020cc17ec581aea4296017de4397d7 100644
--- a/rattail/db/importing/__init__.py
+++ b/rattail/db/importing/__init__.py
@@ -24,6 +24,7 @@
Data Importing
"""
-from .core import Importer, make_importer
+from .core import Importer, make_importer, RecordRenderer
from . import models
from .providers import DataProvider, QueryProvider
+from .handlers import ImportHandler
diff --git a/rattail/db/importing/core.py b/rattail/db/importing/core.py
index bb188b0266af5e08c48d9383448a87b860f5e9c9..815a6edbc6831a1d058261bdcb687b3cb50d814f 100644
--- a/rattail/db/importing/core.py
+++ b/rattail/db/importing/core.py
@@ -130,6 +130,9 @@ class Importer(Object):
if progress:
prog = progress("Importing {0} data".format(self.model_name), count)
+ created = []
+ updated = []
+
affected = 0
keys_seen = set()
for i, src_data in enumerate(records, 1):
@@ -147,6 +150,7 @@ class Importer(Object):
if self.data_differs(inst_data, src_data):
instance = self.get_instance(src_data)
self.update_instance(instance, src_data, inst_data)
+ updated.append(instance)
dirty = True
else:
instance = self.new_instance(src_data)
@@ -155,6 +159,7 @@ class Importer(Object):
self.session.add(instance)
self.session.flush()
log.debug("created new {0} {1}: {2}".format(self.model_name, instance.uuid, instance))
+ created.append(instance)
dirty = True
if dirty:
@@ -169,7 +174,7 @@ class Importer(Object):
if prog:
prog.destroy()
- return affected
+ return created, updated
def setup(self, progress):
"""
@@ -337,3 +342,60 @@ class Importer(Object):
if field in data:
if not inst_data or inst_data[field] != data[field]:
setattr(instance, field, data[field])
+
+
+class RecordRenderer(object):
+ """
+ Record renderer for email notifications sent from data import jobs.
+ """
+
+ def __init__(self, config):
+ self.config = config
+
+ def __call__(self, record):
+ return self.render(record)
+
+ def render(self, record):
+ """
+ Render the given record. Default is to attempt.
+ """
+ key = record.__class__.__name__.lower()
+ renderer = getattr(self, 'render_{0}'.format(key), None)
+ if renderer:
+ return renderer(record)
+
+ label = self.get_label(record)
+ url = self.get_url(record)
+ if url:
+ return '{1}'.format(url, label)
+ return label
+
+ def get_label(self, record):
+ key = record.__class__.__name__.lower()
+ label = getattr(self, 'label_{0}'.format(key), self.label)
+ return label(record)
+
+ def label(self, record):
+ return unicode(record)
+
+ def get_url(self, record):
+ """
+ Fetch / generate a URL for the given data record. You should *not*
+ override this method, but do :meth:`url()` instead.
+ """
+ key = record.__class__.__name__.lower()
+ url = getattr(self, 'url_{0}'.format(key), self.url)
+ return url(record)
+
+ def url(self, record):
+ """
+ Fetch / generate a URL for the given data record.
+ """
+ url = self.config.get('tailbone', 'url')
+ if url:
+ url = url.rstrip('/')
+ name = '{0}s'.format(record.__class__.__name__.lower())
+ if name == 'persons': # FIXME, obviously this is a hack
+ name = 'people'
+ url = '{0}/{1}/{{uuid}}'.format(url, name)
+ return url.format(uuid=record.uuid)
diff --git a/rattail/db/importing/handlers.py b/rattail/db/importing/handlers.py
new file mode 100644
index 0000000000000000000000000000000000000000..6177175d2e64ade5c624382a8809d2fdbf51b957
--- /dev/null
+++ b/rattail/db/importing/handlers.py
@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+################################################################################
+#
+# Rattail -- Retail Software Framework
+# Copyright © 2010-2015 Lance Edgar
+#
+# This file is part of Rattail.
+#
+# Rattail is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Rattail. If not, see .
+#
+################################################################################
+"""
+Import Handlers
+"""
+
+from __future__ import unicode_literals
+
+import logging
+
+from rattail.util import OrderedDict
+
+
+log = logging.getLogger(__name__)
+
+
+class ImportHandler(object):
+ """
+ Base class for all import handlers.
+ """
+
+ def __init__(self, config=None, session=None):
+ self.config = config
+ self.session = session
+ self.importers = self.get_importers()
+
+ def get_importers(self):
+ """
+ Returns a dict of all available importers, where the values are
+ importer factories. All subclasses will want to override this. Note
+ that if you return an ``OrderedDict`` instance, you can affect the
+ ordering of keys in the command line help system, etc.
+ """
+ return {}
+
+ def get_importer_keys(self):
+ """
+ Returns a list of keys corresponding to the available importers.
+ """
+ return list(self.importers.iterkeys())
+
+ def get_importer(self, key):
+ """
+ Returns an importer instance corresponding to the given key.
+ """
+ return self.importers[key](self.config, self.session,
+ **self.get_importer_kwargs(key))
+
+ def get_importer_kwargs(self, key):
+ """
+ Return a dict of kwargs to be used when construcing an importer with
+ the given key.
+ """
+ return {}
+
+ def import_data(self, keys, max_updates=None, progress=None):
+ """
+ Import all data for the given importer keys.
+ """
+ self.before_import()
+ updates = OrderedDict()
+
+ for key in keys:
+ provider = self.get_importer(key)
+ if not provider:
+ log.warning("unknown importer; skipping: {0}".format(repr(key)))
+ continue
+
+ data = provider.get_data(progress=progress)
+ created, updated = provider.importer.import_data(
+ data, provider.supported_fields, provider.key,
+ max_updates=max_updates, progress=progress)
+
+ if hasattr(provider, 'process_deletions'):
+ deleted = provider.process_deletions(data, progress=progress)
+ else:
+ deleted = 0
+
+ log.info("added {0}, updated {1}, deleted {2} {3} records".format(
+ len(created), len(updated), deleted, key))
+ if created or updated or deleted:
+ updates[key] = created, updated, deleted
+
+ self.after_import()
+ return updates
+
+ def before_import(self):
+ return
+
+ def after_import(self):
+ return
diff --git a/rattail/templates/mail/rattail_import_updates.html.mako b/rattail/templates/mail/rattail_import_updates.html.mako
new file mode 100644
index 0000000000000000000000000000000000000000..9f3f5704a1af7d172230ea67808675d58168aaec
--- /dev/null
+++ b/rattail/templates/mail/rattail_import_updates.html.mako
@@ -0,0 +1,52 @@
+
+
+ Data Import Warnings (${command})
+ % if dry_run:
+
+ NOTE: This was a dry run only; no data was harmed
+ in the making of this email.
+
+ % endif
+
+ Generally the periodic data import is expected to be a precaution only, in order
+ to detect and fix Rattail data which falls out of sync from the data authority,
+ e.g. your POS. It is normally expected that proper real-time operation
+ should be enough to keep things in sync; therefore any actual changes
+ which occur as part of the import process are considered "warnings".
+
+
+ The following is a list of changes which occurred during the latest
+ import run. Please investigate at your convenience.
+
+
+ % for model, (created, updated, deleted) in updates.iteritems():
+ -
+ ${model}
+ - ${len(created)} created, ${len(updated)} updated, ${deleted} deleted
+
+ % endfor
+
+ % for model, (created, updated, deleted) in updates.iteritems():
+
+ % for label, records in (('created', created), ('updated', updated)):
+ % if records:
+ % if len(records) == 1:
+ 1 record was ${label}:
+ % else:
+ ${len(records)} records were ${label}:
+ % endif
+
+ % for record in records:
+ - ${render_record(record)}
+ % endfor
+
+ % endif
+ % endfor
+ % if deleted == 1:
+ 1 record was deleted
+ % elif deleted:
+ ${deleted} records were deleted
+ % endif
+ % endfor
+
+