From aa56e4a1894df560e4806ccd096f0cda1c04b6f8 2021-10-03 14:26:58 From: Lance Edgar Date: 2021-10-03 14:26:58 Subject: [PATCH] Add initial version of the "mailmon" daemon and refactor some filemon config etc. to leverage common logic --- diff --git a/rattail/commands/core.py b/rattail/commands/core.py index 983e1769dc0096e3a1593984295ffd72d0196d62..6bcf895158a73de615f20fee950400a816e23737 100644 --- a/rattail/commands/core.py +++ b/rattail/commands/core.py @@ -969,6 +969,43 @@ class FileMonitorCommand(Subcommand): service.delayed_auto_start_service(name) +class MailMonitorCommand(Subcommand): + """ + Interacts with the mail monitor service; called as ``rattail + mailmon``. This command expects a subcommand; one of the + following: + + * ``rattail mailmon start`` + * ``rattail mailmon stop`` + """ + name = 'mailmon' + description = "Manage the mail monitor daemon" + + def add_parser_args(self, parser): + subparsers = parser.add_subparsers(title='subcommands') + + start = subparsers.add_parser('start', help="Start service") + start.set_defaults(subcommand='start') + + stop = subparsers.add_parser('stop', help="Stop service") + stop.set_defaults(subcommand='stop') + + parser.add_argument('-p', '--pidfile', metavar='PATH', default='/var/run/rattail/mailmon.pid', + help="Path to PID file.") + + def run(self, args): + from rattail.mailmon.daemon import MailMonitorDaemon + + daemon = MailMonitorDaemon(args.pidfile, config=self.config) + if args.subcommand == 'stop': + daemon.stop() + else: # start + try: + daemon.start(daemonize=False) + except KeyboardInterrupt: + self.stderr.write("Interrupted.\n") + + class LoadHostDataCommand(Subcommand): """ Loads data from the Rattail host database, if one is configured. diff --git a/rattail/config.py b/rattail/config.py index 916e23727c410ef847cad0f80b6134b74e14ba23..34ce028aab0ddd1d5d31c6066b90d3170c8d0694 100644 --- a/rattail/config.py +++ b/rattail/config.py @@ -38,7 +38,7 @@ import logging.config import six -from rattail.util import load_entry_points, import_module_path +from rattail.util import load_entry_points, import_module_path, load_object from rattail.exceptions import WindowsExtensionsNotInstalled, ConfigurationError from rattail.files import temp_path from rattail.logging import TimeConverter @@ -837,6 +837,30 @@ class ConfigProfile(object): def __init__(self, config, key, **kwargs): self.config = config self.key = key + self.load() + + def load(self): + """ + Read all relevant settings etc. from the config object, + setting attributes on this profile instance as needed. + """ + + def load_defaults(self): + """ + Read all "default" (common) settings from config, for the + current profile. + """ + self.workdir = self._config_string('workdir') + self.stop_on_error = self._config_boolean('stop_on_error', False) + + def load_actions(self): + """ + Read the "actions" from config, for the current profile, and + assign the result to ``self.actions``. + """ + self.actions = [] + for action in self._config_list('actions'): + self.actions.append(self._config_action(action)) @property def section(self): @@ -881,6 +905,99 @@ class ConfigProfile(object): def _config_list(self, option): return parse_list(self._config_string(option)) + def _config_action(self, name): + """ + Retrieve an "action" value from config, for the current + profile. This returns a :class:`ConfigProfileAction` + instance. + """ + from rattail.monitoring import CommandAction + + function = self._config_string('action.{}.func'.format(name)) + class_ = self._config_string('action.{}.class'.format(name)) + cmd = self._config_string('action.{}.cmd'.format(name)) + + specs = [1 if spec else 0 for spec in (function, class_, cmd)] + if sum(specs) != 1: + raise ConfigurationError( + "Monitor profile '{}' (action '{}') must have exactly one of: " + "function, class, command".format(self.key, name)) + + action = ConfigProfileAction() + action.config = self.config + + if function: + action.spec = function + action.action = load_object(action.spec) + elif class_: + action.spec = class_ + action.action = load_object(action.spec)(self.config) + elif cmd: + action.spec = cmd + action.action = CommandAction(self.config, cmd) + + action.args = self._config_list('action.{}.args'.format(name)) + + action.kwargs = {} + pattern = re.compile(r'^{}\.action\.{}\.kwarg\.(?P\w+)$'.format(self.key, name), re.IGNORECASE) + + # TODO: this should not be referencing the config parser directly! + # (but we have no other way yet, to know which options are defined) + # (we should at least allow config to be defined in DB Settings) + # (however that should be optional, since some may not have a DB) + for option in self.config.parser.options(self.section): + match = pattern.match(option) + if match: + action.kwargs[match.group('keyword')] = self.config.get(self.section, option) + + action.retry_attempts = self._config_int('action.{}.retry_attempts'.format(name), minimum=1) + action.retry_delay = self._config_int('action.{}.retry_delay'.format(name), minimum=0) + return action + + +class ConfigProfileAction(object): + """ + Simple class to hold configuration for a particular "action" + defined within a monitor :class:`ConfigProfile`. Each instance + has the following attributes: + + .. attribute:: spec + + The original "spec" string used to obtain the action callable. + + .. attribute:: action + + A reference to the action callable. + + .. attribute:: args + + A sequence of positional arguments to be passed to the callable + (in addition to the file path) when invoking the action. + + .. attribute:: kwargs + + A dictionary of keyword arguments to be passed to the callable + (in addition to the positional arguments) when invoking the + action. + + .. attribute:: retry_attempts + + Number of attempts to make when invoking the action. Defaults + to ``1``, meaning the first attempt will be made but no retries + will happen. + + .. attribute:: retry_delay + + Number of seconds to pause between retry attempts, if + :attr:`retry_attempts` is greater than one. Defaults to ``0``. + """ + spec = None + action = None + args = [] + kwargs = {} + retry_attempts = 1 + retry_delay = 0 + class FreeTDSLoggingFilter(logging.Filter): """ diff --git a/rattail/exceptions.py b/rattail/exceptions.py index da2b0ba00b01de9ff577f6df7fdd656a5a4d0063..45a4d35f715d83d221293be08c3eeb95bd6cab49 100644 --- a/rattail/exceptions.py +++ b/rattail/exceptions.py @@ -179,3 +179,10 @@ class PalmConduitNotRegistered(PalmError): def __str__(self): return "The Rattail Palm conduit is not registered." + + +class StopProcessing(RattailError): + """ + Simple exception to indicate action processing should stop. This + is probably only useful for tests. + """ diff --git a/rattail/filemon/actions.py b/rattail/filemon/actions.py index dfa02ea07820e59005fb32677699f69461a285ed..681b0430134de46fe8e37b14565fd5050a8d8a22 100644 --- a/rattail/filemon/actions.py +++ b/rattail/filemon/actions.py @@ -37,27 +37,18 @@ from traceback import format_exception from rattail.config import parse_bool, parse_list from rattail.mail import send_email +from rattail.monitoring import MonitorAction +from rattail.exceptions import StopProcessing log = logging.getLogger(__name__) -class Action(object): +class Action(MonitorAction): """ Base class for file monitor actions. """ - def __init__(self, config): - self.config = config - - def __call__(self, *args, **kwargs): - """ - This method must be implemented in the subclass; it defines what the - action actually *does*. The file monitor will invoke this method for - all new files which are discovered. - """ - raise NotImplementedError - class CommandAction(Action): """ @@ -92,13 +83,6 @@ class CommandAction(Action): subprocess.check_call(cmd, shell=shell) -class StopProcessing(Exception): - """ - Simple exception to indicate action processing should stop. This is really - only useful for tests. - """ - - def perform_actions(profile): """ Target for action threads. Provides the main loop which checks the queue diff --git a/rattail/filemon/config.py b/rattail/filemon/config.py index 0c30a4e0f3a1b8e16976bbd845f25b0e301d61d8..48c21c2accca66471ae0af5d2003f000b0719fa7 100644 --- a/rattail/filemon/config.py +++ b/rattail/filemon/config.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2019 Lance Edgar +# Copyright © 2010-2021 Lance Edgar # # This file is part of Rattail. # @@ -32,7 +32,7 @@ import getpass import warnings import logging -from rattail.config import parse_list +from rattail.config import parse_list, ConfigProfileAction from rattail.util import load_object from rattail.exceptions import ConfigurationError from rattail.filemon.actions import CommandAction @@ -41,46 +41,8 @@ from rattail.filemon.actions import CommandAction log = logging.getLogger(__name__) -class ProfileAction(object): - """ - Simple class to hold configuration for a particular action defined within a - monitor :class:`Profile`. Each instance has the following attributes: - - .. attribute:: spec - - The original "spec" string used to obtain the action callable. - - .. attribute:: action - - A reference to the action callable. - - .. attribute:: args - - A sequence of positional arguments to be passed to the callable (in - addition to the file path) when invoking the action. - - .. attribute:: kwargs - - A dictionary of keyword arguments to be passed to the callable (in - addition to the positional arguments) when invoking the action. - - .. attribute:: retry_attempts - - Number of attempts to make when invoking the action. Defaults to ``1``, - meaning the first attempt will be made but no retries will happen. - - .. attribute:: retry_delay - - Number of seconds to pause between retry attempts, if - :attr:`retry_attempts` is greater than one. Defaults to ``0``. - """ - - spec = None - action = None - args = [] - kwargs = {} - retry_attempts = 1 - retry_delay = 0 +# TODO: deprecate / remove this +ProfileAction = ConfigProfileAction class Profile(object): @@ -151,8 +113,9 @@ class Profile(object): .. attribute:: actions - List of :class:`ProfileAction` instances representing the actions to be - invoked when new files are discovered. + List of :class:`~rattail.config.ConfigProfileAction` instances + representing the actions to be invoked when new files are + discovered. """ def __init__(self, config, key): @@ -176,8 +139,9 @@ class Profile(object): def _config_action(self, name): """ - Retrieve an "action" value from config, for the current profile. This - returns a :class:`ProfileAction` instance. + Retrieve an "action" value from config, for the current + profile. This returns a + :class:`~rattail.config.ConfigProfileAction` instance. """ function = self._config_string('action.{}.func'.format(name)) class_ = self._config_string('action.{}.class'.format(name)) @@ -189,7 +153,7 @@ class Profile(object): "File monitor profile '{}' (action '{}') must have exactly one of: " "function, class, command".format(self.key, name)) - action = ProfileAction() + action = ConfigProfileAction() action.config = self.config if function: diff --git a/rattail/mailmon/__init__.py b/rattail/mailmon/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4f9f66d4386e2cf3569ec8e3783ea1812732510d --- /dev/null +++ b/rattail/mailmon/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2021 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Mail Monitoring - more specifically, Email Folder Monitoring +""" + +from .actions import MessageAction diff --git a/rattail/mailmon/actions.py b/rattail/mailmon/actions.py new file mode 100644 index 0000000000000000000000000000000000000000..fe9c759ae1307bd9b3ed1849b1de57c077a750e3 --- /dev/null +++ b/rattail/mailmon/actions.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2021 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Mail Monitor Actions +""" + +from __future__ import unicode_literals, absolute_import + +import os +import shutil +import tempfile + +from rattail.monitoring import MonitorAction +from rattail.config import parse_bool +from rattail.files import locking_copy + + +class MessageAction(MonitorAction): + """ + Base class for mailmon message actions. + """ + + def __call__(self, server, msguid, *args, **kwargs): + """ + This method must be implemented in the subclass; it defines + what the action actually *does*. The monitor daemon will + invoke this method for all new messages which are discovered. + + :param server: Reference to the ``imaplib.server`` instance, + which will be connected with an active session. + + :param msguid: UID for the message upon which to act. + """ + raise NotImplementedError + + +def download_message(server, msg_uid, output_dir, locking=False): + """ + Simple action to "download" a message to local filesystem. + + :param output_dir: Path to the folder into which message should be + written. Note that the filename will be like + ``{msg_uid}.eml``. + + :param locking: Flag to indicate that the + :func:`rattail.files.locking_copy()` function should be used to + place the file in its final location. This is useful if you + then also have a rattail filemon watching the ``output_dir``. + """ + if not isinstance(locking, bool): + locking = parse_bool(locking) + + # fetch message data + code, msg_data = server.uid('fetch', msg_uid, '(RFC822)') + if code != 'OK': + raise RuntimeError("IMAP4.fetch() for msg_uid %s returned " + "bad code %s - msg_data is: %s", + msg_uid, code, msg_data) + + # extract message body + # TODO: what do these signify? + assert len(msg_data) == 2 + assert msg_data[1] == ')' + response, msg_body = msg_data[0] + + # figure out where we need to write the file + filename = '{}.eml'.format(msg_uid) + if locking: + tempdir = tempfile.mkdtemp() + path = os.path.join(tempdir, filename) + else: # no locking, write directly to file + path = os.path.join(output_dir, filename) + + # write message to file + with open(path, 'wb') as f: + f.write(msg_body) + + # maybe move temp file to final path + if locking: + locking_copy(path, output_dir) + shutil.rmtree(tempdir) + + +def move_message(server, msguid, newfolder): + """ + Simple action to "move" a message to another IMAP folder, on the + same server. + """ + # copy msg to new folder + code, response = server.uid('COPY', msguid, newfolder) + if code != 'OK': + raise RuntimeError("IMAP.copy(uid={}) returned bad code: {}".format( + msguid, code)) + + # mark old msg as deleted + code, response = server.uid('STORE', msguid, '+FLAGS', '(\Deleted)') + if code != 'OK': + raise RuntimeError("IMAP.store(uid={}) returned bad code: {}".format( + msguid, code)) + + # expunge deleted messages + code, response = server.expunge() + if code != 'OK': + raise RuntimeError("IMAP.expunge() returned bad code: {}".format(code)) diff --git a/rattail/mailmon/config.py b/rattail/mailmon/config.py new file mode 100644 index 0000000000000000000000000000000000000000..bb013ab6606905f542f917895b7bae87d19e2afc --- /dev/null +++ b/rattail/mailmon/config.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2021 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Mail Monitor Configuration +""" + +from __future__ import unicode_literals, absolute_import + +import six + +from rattail.config import ConfigProfile, parse_list +from rattail.exceptions import ConfigurationError + + +class MailMonitorProfile(ConfigProfile): + """ + Simple class to hold configuration for a MailMon "profile". Each + profile determines which email folder(s) will be watched for new + messages, and which action(s) will then be invoked to process the + messages. + """ + section = 'rattail.mailmon' + + def load(self): + + self.imap_server = self._config_string('imap.server') + self.imap_username = self._config_string('imap.username') + self.imap_password = self._config_string('imap.password') + self.imap_folder = self._config_string('imap.folder') + self.imap_unread_only = self._config_boolean('imap.unread_only') + self.imap_delay = self._config_int('imap.delay', default=120) + self.imap_recycle = self._config_int('imap.recycle', default=1200) + + self.max_batch_size = self._config_int('max_batch_size', default=100) + + self.load_defaults() + self.load_actions() + + def validate(self): + """ + Validate the configuration for current profile. + """ + if not self.actions: + raise ConfigurationError("mailmon profile '{}' has no valid " + "actions to invoke".format(self.key)) + + +def load_mailmon_profiles(config): + """ + Load all active mail monitor profiles defined within configuration. + """ + # make sure we have a top-level directive + keys = config.get('rattail.mailmon', 'monitor') + if not keys: + raise ConfigurationError( + "The mail monitor configuration does not specify any profiles " + "to be monitored. Please defined the 'monitor' option within " + "the [rattail.mailmon] section of your config file.") + + monitored = {} + for key in parse_list(keys): + profile = MailMonitorProfile(config, key) + + # only monitor this profile if it validates + try: + profile.validate() + except ConfigurationError as error: + log.warning(six.text_type(error)) + else: + monitored[key] = profile + + return monitored diff --git a/rattail/mailmon/daemon.py b/rattail/mailmon/daemon.py new file mode 100644 index 0000000000000000000000000000000000000000..057981a07dcde3ccf07351c320ff7d20f1294439 --- /dev/null +++ b/rattail/mailmon/daemon.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2021 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Mail Monitor Daemon +""" + +from __future__ import unicode_literals, absolute_import + +import re +import time +import imaplib +from six.moves import queue +# import sys +import logging +# from traceback import format_exception + +import six + +from rattail.daemon import Daemon +from rattail.mailmon.config import load_mailmon_profiles +from rattail.threads import Thread +from rattail.time import make_utc +from rattail.exceptions import StopProcessing +# from rattail.mail import send_email + + +log = logging.getLogger(__name__) + + +class MailMonitorDaemon(Daemon): + """ + Daemon responsible for checking IMAP folders and detecting email + messages, and then invoking actions upon them. + """ + + def run(self): + """ + Starts watcher and worker threads according to configuration. + """ + monitored = load_mailmon_profiles(self.config) + for key, profile in six.iteritems(monitored): + + # create a msg queue for the profile + profile.queue = queue.Queue() + + # create a watcher thread for the IMAP folder + watcher = IMAPWatcher(profile) + name = 'watcher_{}'.format(key) + log.info("starting IMAP watcher thread: %s", name) + thread = Thread(target=watcher, name=name) + thread.daemon = True + thread.start() + + # create an action thread for the profile + name = 'actions-{}'.format(key) + log.debug("starting action thread: %s", name) + thread = Thread(target=perform_actions, name=name, + args=(self.config, watcher)) + thread.daemon = True + thread.start() + + # loop indefinitely. since this is the main thread, the app + # will terminate when this method ends; all other threads are + # "subservient" to this one. + while True: + time.sleep(.01) + + +class IMAPWatcher(object): + """ + Abstraction to make watching an IMAP folder a little more + organized. Instances of this class are used as callable targets + when the daemon starts watcher threads. They are responsible for + polling the IMAP folder and processing any messages found there. + """ + uid_pattern = re.compile(r'^\d+ \(UID (?P\d+)') + + def __init__(self, profile): + self.profile = profile + self.server = None + + def get_uid(self, response): + match = self.uid_pattern.match(response) + if match: + return match.group('uid') + + def __call__(self): + recycled = None + while True: + + if self.server is None: + self.server = imaplib.IMAP4_SSL(self.profile.imap_server) + try: + result = self.server.login(self.profile.imap_username, self.profile.imap_password) + except self.server.error: + log.exception("failed to login to server!") + return + + log.debug("IMAP server login result: %s", result) + recycled = make_utc() + + self.server.select(self.profile.imap_folder) + + try: + self.queue_messages() + except: + log.exception("failed to queue messages!") + if profile.stop_on_error: + break + + time.sleep(self.profile.imap_delay) + + # If recycle time limit has been reached, close and reopen the IMAP connection. + if (make_utc() - recycled).seconds >= self.profile.imap_recycle: + log.debug("recycle time limit reached, disposing of current connection") + self.server.close() + self.server.logout() + self.server = None + + self.server.close() + self.server.logout() + + def queue_messages(self): + """ + Check for new messages in the folder, and queue any found, for + action processing thread. + """ + # maybe look for "all" or maybe just "unread" + if self.profile.imap_unread_only: + criterion = '(UNSEEN)' + else: + criterion = 'ALL' + + # log.debug("invoking IMAP4.search()") + code, items = self.server.uid('search', None, criterion) + if code != 'OK': + raise RuntimeError("IMAP4.search() returned bad code: {}".format(code)) + + # config may dictacte a "max batch size" in which case we will + # only queue so many messages at a time + uids = items[0].split() + if self.profile.max_batch_size: + if len(uids) > self.profile.max_batch_size: + uids = uids[:self.profile.max_batch_size] + + # add message uids to the queue + for uid in uids: + self.profile.queue.put(uid) + + +def perform_actions(config, watcher): + """ + Target for action threads. Provides the main loop which checks + the queue for new messages and invokes actions for each, as they + appear. + """ + profile = watcher.profile + stop = False + while not stop: + + # suspend execution briefly, to avoid consuming so much CPU... + time.sleep(0.01) + + try: + msguid = profile.queue.get_nowait() + except queue.Empty: + pass + except StopProcessing: + stop = True + else: + log.debug("queue contained a msguid: %s", msguid) + for action in profile.actions: + try: + invoke_action(config, watcher, action, msguid) + + except: + # stop processing messages altogether for this + # profile if it is so configured + if profile.stop_on_error: + log.warning("an error was encountered, and config " + "dictates that no more actions should be " + "processed for profile: %s", profile.key) + stop = True + + # either way no more actions should be invoked for + # this particular message + break + + +def invoke_action(config, watcher, action, msguid): + """ + Invoke a single action on a mail message, retrying as necessary. + """ + attempts = 0 + errtype = None + while True: + attempts += 1 + log.debug("invoking action '%s' (attempt #%s of %s) on file: %s", + action.spec, attempts, action.retry_attempts, msguid) + + try: + action.action(watcher.server, msguid, *action.args, **action.kwargs) + + except: + + # if we've reached our final attempt, stop retrying + if attempts >= action.retry_attempts: + # log.debug("attempt #%s failed for action '%s' (giving up) on " + # "msguid: %s", attempts, action.spec, msguid, + # exc_info=True) + log.exception("attempt #%s failed for action '%s' (giving up) on " + "msguid: %s", attempts, action.spec, msguid) + # TODO: add email support + # exc_type, exc, traceback = sys.exc_info() + # send_email(config, 'mailmon_action_error', { + # # 'hostname': socket.gethostname(), + # # 'path': path, + # 'msguid': msguid, + # 'action': action, + # 'attempts': attempts, + # 'error': exc, + # 'traceback': ''.join(format_exception(exc_type, exc, traceback)).strip(), + # }) + raise + + # if this exception is not the first, and is of a + # different type than seen previously, do *not* continue + # to retry + if errtype is not None and not isinstance(error, errtype): + log.exception("new exception differs from previous one(s), " + "giving up on action '%s' for msguid: %s", + action.spec, msguid) + raise + + # record the type of exception seen, and pause for next retry + log.warning("attempt #%s failed for action '%s' on msguid: %s", + attempts, action.spec, msguid, exc_info=True) + errtype = type(error) + log.debug("pausing for %s seconds before making attempt #%s of %s", + action.retry_delay, attempts + 1, action.retry_attempts) + if action.retry_delay: + time.sleep(action.retry_delay) + + else: + # no error, invocation successful + log.debug("attempt #%s succeeded for action '%s' on msguid: %s", + attempts, action.spec, msguid) + break diff --git a/rattail/monitoring.py b/rattail/monitoring.py new file mode 100644 index 0000000000000000000000000000000000000000..ea17142b3c036dfe5932572cd36b628050ab2849 --- /dev/null +++ b/rattail/monitoring.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2021 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Monitoring Library + +This contains misc. common/shared logic for use with multiple types of +monitors, e.g. datasync, filemon etc. +""" + +from __future__ import unicode_literals, absolute_import + +import os +import logging +import subprocess + +from rattail.config import parse_bool, parse_list + + +log = logging.getLogger(__name__) + + +class MonitorAction(object): + """ + Base class for monitor actions. Note that not all actions are + class-based, but the ones which are should probably inherit from + this class. + """ + + def __init__(self, config): + self.config = config + self.app = config.get_app() + self.enum = config.get_enum() + self.model = config.get_model() + + def __call__(self, *args, **kwargs): + """ + This method must be implemented in the subclass; it defines + what the action actually *does*. The monitor daemon will + invoke this method for all new items which are discovered. + """ + raise NotImplementedError + + +class CommandAction(MonitorAction): + """ + Simple action which can execute an arbitrary command, as a + subprocess. This action is meant to be invoked with a particular + file path, which is to be acted upon. + """ + + def __init__(self, config, cmd): + super(CommandAction, self).__init__(config) + self.cmd = cmd + + def __call__(self, path, **kwargs): + """ + Run the command for the given file path. + """ + filename = os.path.basename(path) + shell = parse_bool(kwargs.pop('shell', False)) + + if shell: + # TODO: probably shoudn't use format() b/c who knows what is in + # that command line, that might trigger errors + cmd = self.cmd.format(path=path, filename=filename) + + else: + cmd = [] + for term in parse_list(self.cmd): + term = term.replace('{path}', path) + term = term.replace('{filename}', filename) + cmd.append(term) + + log.debug("final command to run is: %s", cmd) + subprocess.check_call(cmd, shell=shell) diff --git a/rattail/tests/filemon/test_actions.py b/rattail/tests/filemon/test_actions.py index 404e57d1dda8121f5fddc4cbdd8d69fbc1d61c59..71ffe69a0e57eede9b7eac9e863c12008515bd33 100644 --- a/rattail/tests/filemon/test_actions.py +++ b/rattail/tests/filemon/test_actions.py @@ -11,9 +11,9 @@ from unittest import TestCase from mock import Mock, patch, call -from rattail.config import make_config, RattailConfig +from rattail.config import make_config, RattailConfig, ConfigProfileAction from rattail.filemon import actions -from rattail.filemon.config import Profile, ProfileAction +from rattail.filemon.config import Profile class TestAction(TestCase): @@ -110,7 +110,7 @@ class TestPerformActions(TestCase): class TestInvokeAction(TestCase): def setUp(self): - self.action = ProfileAction() + self.action = ConfigProfileAction() self.action.config = RattailConfig() self.action.action = Mock(return_value=None) self.action.retry_attempts = 6 diff --git a/rattail/tests/test_monitoring.py b/rattail/tests/test_monitoring.py new file mode 100644 index 0000000000000000000000000000000000000000..c21190b11457182cdb7b51ab8b9a5e8df7a74df2 --- /dev/null +++ b/rattail/tests/test_monitoring.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8; -*- + +from __future__ import unicode_literals, absolute_import + +import unittest + +from mock import patch, Mock + +from rattail import monitoring +from rattail.config import RattailConfig + + +class TestMonitorAction(unittest.TestCase): + + def setUp(self): + self.config = RattailConfig() + + def test_attributes(self): + action = monitoring.MonitorAction(self.config) + self.assertIs(action.config, self.config) + self.assertTrue(hasattr(action, 'app')) + + def test_not_implemented(self): + action = monitoring.MonitorAction(self.config) + self.assertRaises(NotImplementedError, action) + + +class TestCommandAction(unittest.TestCase): + + def setUp(self): + self.config = RattailConfig() + + def test_attributes(self): + action = monitoring.CommandAction(self.config, "echo test") + self.assertIs(action.config, self.config) + self.assertTrue(hasattr(action, 'app')) + self.assertEqual(action.cmd, "echo test") + + @patch('rattail.monitoring.subprocess') + def test_run_invokes_command(self, subprocess): + subprocess.check_call = Mock() + action = monitoring.CommandAction(self.config, "echo {filename}") + action('test.txt') + self.assertEqual(subprocess.check_call.call_count, 1) + # nb. shell=False is a default kwarg + subprocess.check_call.assert_called_with(['echo', 'test.txt'], shell=False) + + @patch('rattail.monitoring.subprocess') + def test_run_with_shell(self, subprocess): + subprocess.check_call = Mock() + action = monitoring.CommandAction(self.config, "echo {filename}") + action('test.txt', shell=True) + self.assertEqual(subprocess.check_call.call_count, 1) + subprocess.check_call.assert_called_with('echo test.txt', shell=True) diff --git a/setup.py b/setup.py index 01b1692d0de8605c5c6a376a6c06e7b693477feb..06a972ad8e56de5dc2692e7810663baa49363c8b 100644 --- a/setup.py +++ b/setup.py @@ -211,6 +211,7 @@ import-ifps = rattail.commands.importing:ImportIFPS import-rattail = rattail.commands.importing:ImportRattail import-sample = rattail.commands.importing:ImportSampleData import-versions = rattail.commands.importing:ImportVersions +mailmon = rattail.commands.core:MailMonitorCommand make-appdir = rattail.commands.core:MakeAppDir make-batch = rattail.commands.batch:MakeBatch make-config = rattail.commands.core:MakeConfig