From 8ac15ea4702996962522f97d69d93dd94c0d7077 2018-01-06 19:19:20 From: Lance Edgar Date: 2018-01-06 19:19:20 Subject: [PATCH] Add first attempt for "importer as batch" feature --- diff --git a/rattail/batch/importer.py b/rattail/batch/importer.py new file mode 100644 index 0000000000000000000000000000000000000000..a2b78eaaefb8223a8bcf77cd77ef1f026a401add --- /dev/null +++ b/rattail/batch/importer.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2018 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Handler for importer batches +""" + +from __future__ import unicode_literals, absolute_import + +import logging + +import six +import sqlalchemy as sa +from sqlalchemy import orm + +from rattail.db import model +from rattail.batch import BatchHandler +from rattail.util import load_object + + +log = logging.getLogger(__name__) + + +class ImporterBatchHandler(BatchHandler): + """ + Handler for importer batches. + """ + batch_model_class = model.ImporterBatch + + def execute(self, batch, user=None, progress=None, **kwargs): + session = orm.object_session(batch) + metadata = sa.MetaData(schema='batch', bind=session.bind) + row_table = sa.Table(batch.row_table, metadata, autoload=True) + + handler = load_object(batch.import_handler_spec)(config=self.config) + handler.runas_user = user + handler.setup() + handler.begin_transaction() + + importer = handler.get_importer(batch.importer_key) + assert importer + importer.setup() + + def process(row, i): + + if row.status_code == self.enum.IMPORTER_BATCH_ROW_STATUS_CREATE: + host_data = {} + for col in row_table.c: + if col.name.startswith('key_'): + field = col.name[4:] + host_data[field] = getattr(row, col.name) + elif col.name.startswith('post_'): + field = col.name[5:] + host_data[field] = getattr(row, col.name) + key = importer.get_key(host_data) + local_object = importer.create_object(key, host_data) + log.debug("created new %s %s: %s", importer.model_name, key, local_object) + + elif row.status_code == self.enum.IMPORTER_BATCH_ROW_STATUS_UPDATE: + host_data = {} + local_data = {} + for col in row_table.c: + if col.name.startswith('key_'): + field = col.name[4:] + host_data[field] = getattr(row, col.name) + local_data[field] = getattr(row, col.name) + elif col.name.startswith('pre_'): + field = col.name[4:] + local_data[field] = getattr(row, col.name) + elif col.name.startswith('post_'): + field = col.name[5:] + host_data[field] = getattr(row, col.name) + key = importer.get_key(host_data) + local_object = importer.get_local_object(key) + local_object = importer.update_object(local_object, host_data, local_data) + log.debug("updated %s %s: %s", importer.model_name, key, local_object) + + elif row.status_code == self.enum.IMPORTER_BATCH_ROW_STATUS_DELETE: + host_data = {} + for col in row_table.c: + if col.name.startswith('key_'): + field = col.name[4:] + host_data[field] = getattr(row, col.name) + key = importer.get_key(host_data) + local_object = importer.get_local_object(key) + text = six.text_type(local_object) + importer.delete_object(local_object) + log.debug("deleted %s %s: %s", importer.model_name, key, text) + + rows = session.query(row_table) + self.progress_loop(process, rows, progress, + message="Executing import / export batch") + + importer.teardown() + handler.teardown() + handler.commit_transaction() + return True diff --git a/rattail/commands/importing.py b/rattail/commands/importing.py index 9da1d710d93ac4074ad3148bed908650f4a90211..7ef12a2dfa5a81f419bcdb06bfbcbde5094c235f 100644 --- a/rattail/commands/importing.py +++ b/rattail/commands/importing.py @@ -106,6 +106,11 @@ class ImportSubcommand(Subcommand): doc += " Supported models are: ({})".format(', '.join(handler.get_importer_keys())) parser.add_argument('models', nargs='*', metavar='MODEL', help=doc) + # make batches + parser.add_argument('--make-batches', action='store_true', + help="If specified, make new Import / Export Batches instead of " + "performing an actual (possibly dry-run) import.") + # fields / exclude parser.add_argument('--fields', help="List of fields which should be included in the import. " @@ -160,6 +165,7 @@ class ImportSubcommand(Subcommand): "a given import task should stop. Note that this applies on a per-model " "basis and not overall.") + # TODO: deprecate --batch, replace with --batch-size ? # batch size parser.add_argument('--batch', type=int, dest='batch_size', metavar='SIZE', default=200, help="Split work to be done into batches, with the specified number of " @@ -177,11 +183,14 @@ class ImportSubcommand(Subcommand): # dry run? parser.add_argument('--dry-run', action='store_true', help="Go through the full motions and allow logging etc. to " - "occur, but rollback (abort) the transaction at the end.") + "occur, but rollback (abort) the transaction at the end. " + "Note that this flag is ignored if --make-batches is specified.") def run(self, args): - log.info("begin `{} {}` for data models: {}".format( - self.parent_name, self.name, ', '.join(args.models or ["(ALL)"]))) + log.info("begin `%s %s` for data models: %s", + self.parent_name, + self.name, + ', '.join(args.models) if args.models else "(ALL)") handler = self.get_handler(args=args) models = args.models or handler.get_default_keys() @@ -190,7 +199,6 @@ class ImportSubcommand(Subcommand): log.debug("args are: {}".format(args)) kwargs = { - 'dry_run': args.dry_run, 'warnings': args.warnings, 'fields': parse_list(args.fields), 'exclude_fields': parse_list(args.exclude_fields), @@ -204,7 +212,16 @@ class ImportSubcommand(Subcommand): 'progress': self.progress, 'args': args, } - handler.import_data(*models, **kwargs) + if args.make_batches: + kwargs.update({ + 'runas_user': self.get_runas_user(), + }) + handler.make_batches(*models, **kwargs) + else: + kwargs.update({ + 'dry_run': args.dry_run, + }) + handler.import_data(*models, **kwargs) # TODO: should this logging happen elsewhere / be customizable? if args.dry_run: diff --git a/rattail/db/alembic/versions/b82daacc86b7_add_importer_batch.py b/rattail/db/alembic/versions/b82daacc86b7_add_importer_batch.py new file mode 100644 index 0000000000000000000000000000000000000000..2cd9a4ee460778a8af98db6799f92bc5c1ff54a7 --- /dev/null +++ b/rattail/db/alembic/versions/b82daacc86b7_add_importer_batch.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8; -*- +"""add importer batch + +Revision ID: b82daacc86b7 +Revises: a809caf23cf0 +Create Date: 2017-12-29 00:16:43.897114 + +""" + +from __future__ import unicode_literals, absolute_import + +# revision identifiers, used by Alembic. +revision = 'b82daacc86b7' +down_revision = u'0c91cf7d557b' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +import rattail.db.types + + + +def upgrade(): + + # batch_importer + op.create_table('batch_importer', + sa.Column('uuid', sa.String(length=32), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=255), nullable=True), + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('created_by_uuid', sa.String(length=32), nullable=False), + sa.Column('cognized', sa.DateTime(), nullable=True), + sa.Column('cognized_by_uuid', sa.String(length=32), nullable=True), + sa.Column('rowcount', sa.Integer(), nullable=True), + sa.Column('complete', sa.Boolean(), nullable=True), + sa.Column('executed', sa.DateTime(), nullable=True), + sa.Column('executed_by_uuid', sa.String(length=32), nullable=True), + sa.Column('purge', sa.Date(), nullable=True), + sa.Column('notes', sa.Text(), nullable=True), + sa.Column('status_code', sa.Integer(), nullable=True), + sa.Column('status_text', sa.String(length=255), nullable=True), + sa.Column('row_table', sa.String(length=255), nullable=False), + sa.Column('batch_handler_spec', sa.String(length=255), nullable=True), + sa.Column('import_handler_spec', sa.String(length=255), nullable=False), + sa.Column('host_title', sa.String(length=255), nullable=False), + sa.Column('local_title', sa.String(length=255), nullable=False), + sa.Column('importer_key', sa.String(length=100), nullable=False), + sa.ForeignKeyConstraint(['cognized_by_uuid'], [u'user.uuid'], name=u'batch_importer_fk_cognized_by'), + sa.ForeignKeyConstraint(['created_by_uuid'], [u'user.uuid'], name=u'batch_importer_fk_created_by'), + sa.ForeignKeyConstraint(['executed_by_uuid'], [u'user.uuid'], name=u'batch_importer_fk_executed_by'), + sa.PrimaryKeyConstraint('uuid') + ) + + +def downgrade(): + + # batch_importer + op.drop_table('batch_importer') diff --git a/rattail/db/model/__init__.py b/rattail/db/model/__init__.py index 8e20710b558b02ff9958921ec9e4018b39125111..4588dd1ae21f05990a1814af2248e83472a2e448 100644 --- a/rattail/db/model/__init__.py +++ b/rattail/db/model/__init__.py @@ -59,6 +59,7 @@ from .upgrades import Upgrade, UpgradeRequirement from .exports import ExportMixin from .reports import ReportOutput from .batch import BatchMixin, BaseFileBatchMixin, FileBatchMixin, BatchRowMixin, ProductBatchRowMixin +from .batch.dynamic import DynamicBatchMixin, ImporterBatch from .batch.handheld import HandheldBatch, HandheldBatchRow from .batch.inventory import InventoryBatch, InventoryBatchRow from .batch.labels import LabelBatch, LabelBatchRow diff --git a/rattail/db/model/batch/dynamic.py b/rattail/db/model/batch/dynamic.py new file mode 100644 index 0000000000000000000000000000000000000000..9ff52a89b748c7612e9b8347b26c5595a8896e78 --- /dev/null +++ b/rattail/db/model/batch/dynamic.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2018 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Dynamic Batches +""" + +from __future__ import unicode_literals, absolute_import + +import sqlalchemy as sa + +from rattail.db.model import Base, BatchMixin + + +class DynamicBatchMixin(BatchMixin): + """ + Mixin for all dynamic batch (header) classes. + """ + + row_table = sa.Column(sa.String(length=255), nullable=False, doc=""" + Name of the row data table for the batch. This will typically be a UUID + and the table will exist within the 'batch' schema in the PostgreSQL DB. + """) + + # TODO: should nullable be False? + batch_handler_spec = sa.Column(sa.String(length=255), nullable=True, doc=""" + Object spec for the batch handler. + """) + + +class ImporterBatch(DynamicBatchMixin, Base): + """ + Dynamic batch for use with arbitrary data importers. + """ + __tablename__ = 'batch_importer' + batch_key = 'importer' + + import_handler_spec = sa.Column(sa.String(length=255), nullable=False, doc=""" + Object spec for the import handler. + """) + + host_title = sa.Column(sa.String(length=255), nullable=False, doc=""" + Host title for the import handler. + """) + + local_title = sa.Column(sa.String(length=255), nullable=False, doc=""" + Local title for the import handler. + """) + + importer_key = sa.Column(sa.String(length=100), nullable=False, doc=""" + Importer "key" - must be valid within context of the import handler. + """) diff --git a/rattail/enum.py b/rattail/enum.py index 31f4738af7c24d8f884c7c481049998072d08594..7d14f2ad6d0e91db4bfdaf556fecb1177598e521 100644 --- a/rattail/enum.py +++ b/rattail/enum.py @@ -94,6 +94,19 @@ HANDHELD_DEVICE_TYPE = { } +IMPORTER_BATCH_ROW_STATUS_NOCHANGE = 0 +IMPORTER_BATCH_ROW_STATUS_CREATE = 1 +IMPORTER_BATCH_ROW_STATUS_UPDATE = 2 +IMPORTER_BATCH_ROW_STATUS_DELETE = 3 + +IMPORTER_BATCH_ROW_STATUS = { + IMPORTER_BATCH_ROW_STATUS_NOCHANGE : "no change", + IMPORTER_BATCH_ROW_STATUS_CREATE : "create", + IMPORTER_BATCH_ROW_STATUS_UPDATE : "update", + IMPORTER_BATCH_ROW_STATUS_DELETE : "delete", +} + + INVENTORY_MODE_REPLACE = 1 INVENTORY_MODE_REPLACE_ADJUST = 2 INVENTORY_MODE_ADJUST = 3 diff --git a/rattail/importing/handlers.py b/rattail/importing/handlers.py index 74d3baa06115a0185e7afc8e7975804da231e0a4..dbcec0383e2b8b37b81c2baffcf6b9eb196f57ef 100644 --- a/rattail/importing/handlers.py +++ b/rattail/importing/handlers.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2017 Lance Edgar +# Copyright © 2010-2018 Lance Edgar # # This file is part of Rattail. # @@ -31,9 +31,11 @@ import logging import six import humanize +import sqlalchemy as sa +from rattail.core import get_uuid from rattail.time import make_utc -from rattail.util import OrderedDict +from rattail.util import OrderedDict, get_object_spec, progress_loop from rattail.mail import send_email @@ -102,6 +104,212 @@ class ImportHandler(object): """ return kwargs + def make_batches(self, *keys, **kwargs): + """ + Make new import/export batch for each specified model key. + """ + from rattail.db import Session + + session = Session() + model = self.config.get_model() + metadata = sa.MetaData(schema='batch', bind=session.bind) + user = session.merge(kwargs['runas_user']) + handler_spec = self.config.get('rattail.batch', 'importer.handler', + default='rattail.batch.importer:ImporterBatchHandler') + + self.progress = kwargs.pop('progress', getattr(self, 'progress', None)) + + self.setup() + self.begin_transaction() + + for key in keys: + + importer = self.get_importer(key, **kwargs) + if importer and importer.batches_supported: + log.info("making batch for model: %s", key) + importer._handler_key = key + + batch = model.ImporterBatch() + batch.uuid = get_uuid() + batch.created_by = user + batch.batch_handler_spec = handler_spec + batch.import_handler_spec = get_object_spec(self) + batch.host_title = self.host_title + batch.local_title = self.local_title + batch.importer_key = key + batch.rowcount = 0 + + batch.description = "{} -> {} for {}".format( + batch.host_title, + batch.local_title, + batch.importer_key) + + batch.row_table = batch.uuid + + session.add(batch) + session.flush() + + row_table = self.make_row_table(metadata, importer, batch) + self.populate_row_table(session, importer, batch, row_table) + + elif importer: + log.info("batches not supported for importer: %s", key) + + else: + log.warning("skipping unknown model: %s", key) + + self.teardown() + # TODO: necessary? + # self.rollback_transaction() + # self.commit_transaction() + + session.commit() + session.close() + + def make_row_table(self, metadata, importer, batch): + columns = [ + sa.Column('uuid', sa.String(length=32), nullable=False, primary_key=True), + sa.Column('sequence', sa.Integer(), nullable=False), + sa.Column('object_key', sa.String(length=255), nullable=False, default=''), + sa.Column('object_str', sa.String(length=255), nullable=False, default=''), + ] + + for field in importer.fields: + typ = importer.field_coltypes.get(field) + + if not typ and importer.model_class: + mapper = sa.inspect(importer.model_class) + if mapper.has_property(field): + prop = mapper.get_property(field) + if prop: + assert len(prop.columns) == 1, "multiple columns ({}) unsupported: {}.{}".format( + len(prop.columns), batch.importer_key, field) + typ = prop.columns[0].type + + if not typ: + typ = sa.String(length=255) + + if field in importer.key: + columns.append(sa.Column('key_{}'.format(field), typ)) + else: + for prefix in ('pre', 'post'): + columns.append(sa.Column('{}_{}'.format(prefix, field), typ)) + + columns.extend([ + sa.Column('status_code', sa.Integer(), nullable=False), + sa.Column('status_text', sa.String(length=255), nullable=True), + ]) + + row_table = sa.Table(batch.row_table, metadata, *columns) + row_table.create() + return row_table + + def populate_row_table(self, session, importer, batch, row_table): + importer.now = make_utc(tzinfo=True) + importer.setup() + + # obtain host data + host_data = importer.normalize_host_data() + host_data, unique = importer.unique_data(host_data) + if not host_data: + return + + # cache local data if appropriate + if importer.caches_local_data: + importer.cached_local_data = importer.cache_local_data(host_data) + + # create and/or update + if importer.create or importer.update: + self._populate_create_update(session, importer, batch, row_table, host_data) + + # delete + if importer.delete: + self._populate_delete(session, importer, batch, row_table, host_data, set(unique)) + + def _populate_delete(self, session, importer, batch, row_table, host_data, host_keys): + deleting = importer.get_deletion_keys() - host_keys + + def delete(key, i): + cached = importer.cached_local_data.pop(key) + local_data = cached['data'] + local_data['_object_str'] = six.text_type(cached['object']) + sequence = batch.rowcount + 1 + self.make_batch_row(session, importer, row_table, sequence, None, local_data, + status_code=self.enum.IMPORTER_BATCH_ROW_STATUS_DELETE) + batch.rowcount += 1 + + progress_loop(delete, sorted(deleting), self.progress, + message="Deleting {} data".format(importer.model_name)) + + def _populate_create_update(self, session, importer, batch, row_table, data): + + def record(host_data, i): + + # fetch local object, using key from host data + key = importer.get_key(host_data) + local_object = importer.get_local_object(key) + status_code = self.enum.IMPORTER_BATCH_ROW_STATUS_NOCHANGE + status_text = None + make_row = False + + # if we have a local object, but its data differs from host, make an update record + if local_object and importer.update: + make_row = True + local_data = importer.normalize_local_object(local_object) + diffs = importer.data_diffs(local_data, host_data) + if diffs: + status_code = self.enum.IMPORTER_BATCH_ROW_STATUS_UPDATE + status_text = ','.join(diffs) + + # if we did not yet have a local object, make a create record + elif not local_object and importer.create: + make_row = True + local_data = None + status_code = self.enum.IMPORTER_BATCH_ROW_STATUS_CREATE + + if make_row: + sequence = batch.rowcount + 1 + self.make_batch_row(session, importer, row_table, sequence, host_data, local_data, + status_code=status_code, status_text=status_text) + batch.rowcount += 1 + + progress_loop(record, data, self.progress, + message="Populating batch for {}".format(importer._handler_key)) + + def make_batch_row(self, session, importer, row_table, sequence, host_data, local_data, status_code=None, status_text=None): + values = { + 'uuid': get_uuid(), + 'sequence': sequence, + 'object_str': '', + 'status_code': status_code, + 'status_text': status_text, + } + + if host_data: + if '_object_str' in host_data: + values['object_str'] = host_data['_object_str'] + elif '_host_object' in host_data: + values['object_str'] = six.text_type(host_data['_host_object']) + values['object_key'] = ','.join([host_data[f] for f in importer.key]) + elif local_data: + if '_object_str' in local_data: + values['object_str'] = local_data['_object_str'] + elif '_object' in local_data: + values['object_str'] = six.text_type(local_data['_object']) + values['object_key'] = ','.join([local_data[f] for f in importer.key]) + + for field in importer.fields: + if field in importer.key: + data = host_data or local_data + values['key_{}'.format(field)] = data[field] + else: + if host_data and field in host_data: + values['post_{}'.format(field)] = host_data[field] + if local_data and field in local_data: + values['pre_{}'.format(field)] = local_data[field] + + session.execute(row_table.insert(values)) + def import_data(self, *keys, **kwargs): """ Import all data for the given importer/model keys. diff --git a/rattail/importing/importers.py b/rattail/importing/importers.py index 252dbb507e0904cb20ef9b786b391c78b228cf4c..8f05ac4572f96273a95e129aea969ccffa34de0a 100644 --- a/rattail/importing/importers.py +++ b/rattail/importing/importers.py @@ -82,6 +82,18 @@ class Importer(object): host_system_title = None local_system_title = None + # TODO + # Whether or not the registered "importer" batch handler is able to handle + # batches for this importer (and/or, whether this importer is able to + # provide what's needed for the same). + batches_supported = False + + # TODO + # If ``batches_supported`` is true, this should contain SQLAlchemy + # ``Column`` instance overrides, keyed by fieldname. Any field not + # represented here will be given the default column type (string). + field_coltypes = {} + def __init__(self, config=None, key=None, fields=None, exclude_fields=None, **kwargs): self.config = config self.enum = config.get_enum() if config else None @@ -139,6 +151,20 @@ class Importer(object): factory = factory or self.progress return progress_loop(func, items, factory, **kwargs) + def unique_data(self, host_data): + # Prune duplicate keys from host/source data. This is for the sake of + # sanity since duplicates typically lead to a ping-pong effect, where a + # "clean" (change-less) import is impossible. + unique = OrderedDict() + for data in host_data: + key = self.get_key(data) + if key in unique: + log.warning("duplicate records detected from {} for key: {}".format( + self.host_system_title, key)) + else: + unique[key] = data + return list(unique.values()), unique + def import_data(self, host_data=None, now=None, **kwargs): """ Import some data! This is the core body of logic for that, regardless @@ -156,19 +182,7 @@ class Importer(object): # Get complete set of normalized host data. if host_data is None: host_data = self.normalize_host_data() - - # Prune duplicate keys from host/source data. This is for the sake of - # sanity since duplicates typically lead to a ping-pong effect, where a - # "clean" (change-less) import is impossible. - unique = OrderedDict() - for data in host_data: - key = self.get_key(data) - if key in unique: - log.warning("duplicate records detected from {} for key: {}".format( - self.host_system_title, key)) - else: - unique[key] = data - host_data = list(unique.itervalues()) + host_data, unique = self.unique_data(host_data) # Cache local data if appropriate. if self.caches_local_data: @@ -251,6 +265,67 @@ class Importer(object): self.flush_create_update_final() return created, updated + # def _populate_create_update(self, row_table, data): + # """ + # Populate create and/or update records for the given batch row table, + # according to the given host data set. + # """ + # created = [] + # updated = [] + # # count = len(data) + # # if not count: + # # return created, updated + + # def record(host_data, i): + + # # fetch local object, using key from host data + # key = self.get_key(host_data) + # local_object = self.get_local_object(key) + + # # if we have a local object, but its data differs from host, make an update record + # if local_object and self.update: + # local_data = self.normalize_local_object(local_object) + # diffs = self.data_diffs(local_data, host_data) + # if diffs: + # log.debug("fields '{}' differed for local data: {}, host data: {}".format( + # ','.join(diffs), local_data, host_data)) + # local_object = self.update_object(local_object, host_data, local_data) + # updated.append((local_object, local_data, host_data)) + # if self.max_update and len(updated) >= self.max_update: + # log.warning("max of {} *updated* records has been reached; stopping now".format(self.max_update)) + # raise ImportLimitReached() + # if self.max_total and (len(created) + len(updated)) >= self.max_total: + # log.warning("max of {} *total changes* has been reached; stopping now".format(self.max_total)) + # raise ImportLimitReached() + + # # if we did not yet have a local object, make a create record + # elif not local_object and self.create: + # local_object = self.create_object(key, host_data) + # if local_object: + # log.debug("created new {} {}: {}".format(self.model_name, key, local_object)) + # created.append((local_object, host_data)) + # if self.caches_local_data and self.cached_local_data is not None: + # self.cached_local_data[key] = {'object': local_object, 'data': self.normalize_local_object(local_object)} + # if self.max_create and len(created) >= self.max_create: + # log.warning("max of {} *created* records has been reached; stopping now".format(self.max_create)) + # raise ImportLimitReached() + # if self.max_total and (len(created) + len(updated)) >= self.max_total: + # log.warning("max of {} *total changes* has been reached; stopping now".format(self.max_total)) + # raise ImportLimitReached() + # else: + # log.debug("did NOT create new {} for key: {}".format(self.model_name, key)) + + # # flush changes every so often + # if not self.batch_size or (len(created) + len(updated)) % self.batch_size == 0: + # self.flush_create_update() + + # try: + # self.progress_loop(record, data, message="Importing {} data".format(self.model_name)) + # except ImportLimitReached: + # pass + # # self.flush_create_update_final() + # return created, updated + def flush_create_update(self): """ Perform any steps necessary to "flush" the create/update changes which