rattail-project/rattail Changeset

Changeset - aa56e4a1894d

Parent rev.

Child rev.

[Not reviewed]

0 7 6

Lance Edgar (lance) - 3 years ago 2021-10-03 14:26:58
lance@edbob.org

Add initial version of the "mailmon" daemon

and refactor some filemon config etc. to leverage common logic

13 files changed with 840 insertions and 70 deletions:

rattail/commands/core.py

rattail/config.py

118

rattail/exceptions.py

rattail/filemon/actions.py

rattail/filemon/config.py

rattail/mailmon/__init__.py

rattail/mailmon/actions.py

124

rattail/mailmon/config.py

rattail/mailmon/daemon.py

268

rattail/monitoring.py

rattail/tests/filemon/test_actions.py

rattail/tests/test_monitoring.py

setup.py

0 comments (0 inline, 0 general)

rattail/commands/core.py

➞

Show inline comments

@@ @@ -966,12 +966,49 @@ class FileMonitorCommand(Subcommand): @@
         if args.subcommand == 'install' and args.auto_start:
             if platform.release() == '7':
                 name = filemon.RattailFileMonitor._svc_name_
                 service.delayed_auto_start_service(name)
 class MailMonitorCommand(Subcommand):
     """
     Interacts with the mail monitor service; called as ``rattail
     mailmon``.  This command expects a subcommand; one of the
     following:
     * ``rattail mailmon start``
     * ``rattail mailmon stop``
     """
     name = 'mailmon'
     description = "Manage the mail monitor daemon"
     def add_parser_args(self, parser):
         subparsers = parser.add_subparsers(title='subcommands')
         start = subparsers.add_parser('start', help="Start service")
         start.set_defaults(subcommand='start')
         stop = subparsers.add_parser('stop', help="Stop service")
         stop.set_defaults(subcommand='stop')
         parser.add_argument('-p', '--pidfile', metavar='PATH', default='/var/run/rattail/mailmon.pid',
                             help="Path to PID file.")
     def run(self, args):
         from rattail.mailmon.daemon import MailMonitorDaemon
         daemon = MailMonitorDaemon(args.pidfile, config=self.config)
         if args.subcommand == 'stop':
             daemon.stop()
         else: # start
             try:
                 daemon.start(daemonize=False)
             except KeyboardInterrupt:
                 self.stderr.write("Interrupted.\n")
 class LoadHostDataCommand(Subcommand):
     """
     Loads data from the Rattail host database, if one is configured.
     """
     name = 'load-host-data'

rattail/config.py

➞

Show inline comments

@@ @@ -35,13 +35,13 @@ from six.moves import configparser @@
 import warnings
 import logging
 import logging.config
 import six
 from rattail.util import load_entry_points, import_module_path
+from rattail.util import load_entry_points, import_module_path, load_object
 from rattail.exceptions import WindowsExtensionsNotInstalled, ConfigurationError
 from rattail.files import temp_path
 from rattail.logging import TimeConverter
 from rattail.util import prettify
@@ @@ -834,12 +834,36 @@ class ConfigProfile(object): @@
        config section.
     """
     def __init__(self, config, key, **kwargs):
         self.config = config
         self.key = key
         self.load()
     def load(self):
         """
         Read all relevant settings etc. from the config object,
         setting attributes on this profile instance as needed.
         """
     def load_defaults(self):
         """
         Read all "default" (common) settings from config, for the
         current profile.
         """
         self.workdir = self._config_string('workdir')
         self.stop_on_error = self._config_boolean('stop_on_error', False)
     def load_actions(self):
         """
         Read the "actions" from config, for the current profile, and
         assign the result to ``self.actions``.
         """
         self.actions = []
         for action in self._config_list('actions'):
             self.actions.append(self._config_action(action))
     @property
     def section(self):
         """
         Each subclass of ``ConfigProfile`` must define this.
         """
@@ @@ -878,12 +902,105 @@ class ConfigProfile(object): @@
         return value
     def _config_list(self, option):
         return parse_list(self._config_string(option))
     def _config_action(self, name):
         """
         Retrieve an "action" value from config, for the current
         profile.  This returns a :class:`ConfigProfileAction`
         instance.
         """
         from rattail.monitoring import CommandAction
         function = self._config_string('action.{}.func'.format(name))
         class_ = self._config_string('action.{}.class'.format(name))
         cmd = self._config_string('action.{}.cmd'.format(name))
         specs = [1 if spec else 0 for spec in (function, class_, cmd)]
         if sum(specs) != 1:
             raise ConfigurationError(
                 "Monitor profile '{}' (action '{}') must have exactly one of: "
                 "function, class, command".format(self.key, name))
         action = ConfigProfileAction()
         action.config = self.config
         if function:
             action.spec = function
             action.action = load_object(action.spec)
         elif class_:
             action.spec = class_
             action.action = load_object(action.spec)(self.config)
         elif cmd:
             action.spec = cmd
             action.action = CommandAction(self.config, cmd)
         action.args = self._config_list('action.{}.args'.format(name))
         action.kwargs = {}
         pattern = re.compile(r'^{}\.action\.{}\.kwarg\.(?P<keyword>\w+)$'.format(self.key, name), re.IGNORECASE)
         # TODO: this should not be referencing the config parser directly!
         # (but we have no other way yet, to know which options are defined)
         # (we should at least allow config to be defined in DB Settings)
         # (however that should be optional, since some may not have a DB)
         for option in self.config.parser.options(self.section):
             match = pattern.match(option)
             if match:
                 action.kwargs[match.group('keyword')] = self.config.get(self.section, option)
         action.retry_attempts = self._config_int('action.{}.retry_attempts'.format(name), minimum=1)
         action.retry_delay = self._config_int('action.{}.retry_delay'.format(name), minimum=0)
         return action
 class ConfigProfileAction(object):
     """
     Simple class to hold configuration for a particular "action"
     defined within a monitor :class:`ConfigProfile`.  Each instance
     has the following attributes:
     .. attribute:: spec
        The original "spec" string used to obtain the action callable.
     .. attribute:: action
        A reference to the action callable.
     .. attribute:: args
        A sequence of positional arguments to be passed to the callable
        (in addition to the file path) when invoking the action.
     .. attribute:: kwargs
        A dictionary of keyword arguments to be passed to the callable
        (in addition to the positional arguments) when invoking the
        action.
     .. attribute:: retry_attempts
        Number of attempts to make when invoking the action.  Defaults
        to ``1``, meaning the first attempt will be made but no retries
        will happen.
     .. attribute:: retry_delay
        Number of seconds to pause between retry attempts, if
        :attr:`retry_attempts` is greater than one.  Defaults to ``0``.
     """
     spec = None
     action = None
     args = []
     kwargs = {}
     retry_attempts = 1
     retry_delay = 0
 class FreeTDSLoggingFilter(logging.Filter):
     """
     Custom logging filter, to suppress certain "write to server failed"
     messages relating to FreeTDS database connections.  They seem harmless and
     just cause unwanted error emails.

rattail/exceptions.py

➞

Show inline comments

@@ @@ -176,6 +176,13 @@ class PalmConduitAlreadyRegistered(PalmError): @@
 @six.python_2_unicode_compatible
 class PalmConduitNotRegistered(PalmError):
     def __str__(self):
         return "The Rattail Palm conduit is not registered."
 class StopProcessing(RattailError):
     """
     Simple exception to indicate action processing should stop.  This
     is probably only useful for tests.
     """

rattail/filemon/actions.py

➞

Show inline comments

 import subprocess
 import logging
 from traceback import format_exception
 from rattail.config import parse_bool, parse_list
 from rattail.mail import send_email
 from rattail.monitoring import MonitorAction
 from rattail.exceptions import StopProcessing
 log = logging.getLogger(__name__)
-class Action(object):
+class Action(MonitorAction):
     """
     Base class for file monitor actions.
     """
     def __init__(self, config):
         self.config = config
     def __call__(self, *args, **kwargs):
         """
         This method must be implemented in the subclass; it defines what the
         action actually *does*.  The file monitor will invoke this method for
         all new files which are discovered.
         """
         raise NotImplementedError
 class CommandAction(Action):
     """
     Simple file monitor action which can execute a command as a subprocess.
     """
                 cmd.append(term)
         log.debug("final command to run is: %s", cmd)
         subprocess.check_call(cmd, shell=shell)
 class StopProcessing(Exception):
     """
     Simple exception to indicate action processing should stop.  This is really
     only useful for tests.
     """
 def perform_actions(profile):
     """
     Target for action threads.  Provides the main loop which checks the queue
     for new files and invokes actions for each, as they appear.
     """

rattail/filemon/config.py

➞

Show inline comments

 # -*- coding: utf-8; -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2019 Lance Edgar
+#  Copyright © 2010-2021 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU General Public License as published by the Free Software
 #  Foundation, either version 3 of the License, or (at your option) any later
@@ @@ -29,61 +29,23 @@ from __future__ import unicode_literals, absolute_import @@
 import os
 import re
 import getpass
 import warnings
 import logging
 from rattail.config import parse_list
+from rattail.config import parse_list, ConfigProfileAction
 from rattail.util import load_object
 from rattail.exceptions import ConfigurationError
 from rattail.filemon.actions import CommandAction
 log = logging.getLogger(__name__)
 class ProfileAction(object):
     """
     Simple class to hold configuration for a particular action defined within a
     monitor :class:`Profile`.  Each instance has the following attributes:
     .. attribute:: spec
        The original "spec" string used to obtain the action callable.
     .. attribute:: action
        A reference to the action callable.
     .. attribute:: args
        A sequence of positional arguments to be passed to the callable (in
        addition to the file path) when invoking the action.
     .. attribute:: kwargs
        A dictionary of keyword arguments to be passed to the callable (in
        addition to the positional arguments) when invoking the action.
     .. attribute:: retry_attempts
        Number of attempts to make when invoking the action.  Defaults to ``1``,
        meaning the first attempt will be made but no retries will happen.
     .. attribute:: retry_delay
        Number of seconds to pause between retry attempts, if
        :attr:`retry_attempts` is greater than one.  Defaults to ``0``.
     """
     spec = None
     action = None
     args = []
     kwargs = {}
     retry_attempts = 1
     retry_delay = 0
 # TODO: deprecate / remove this
 ProfileAction = ConfigProfileAction
 class Profile(object):
     """
     Simple class to hold configuration for a file monitor "profile".  The
     profile determines which folders to watch, which actions to take when new
@@ @@ -148,14 +110,15 @@ class Profile(object): @@
        If this is ``False`` (the default), then a single file error will only
        cause processing to halt for that particular file.  Subsequent files
        which appear will still be processed as normal.
     .. attribute:: actions
        List of :class:`ProfileAction` instances representing the actions to be
        invoked when new files are discovered.
        List of :class:`~rattail.config.ConfigProfileAction` instances
        representing the actions to be invoked when new files are
        discovered.
     """
     def __init__(self, config, key):
         self.config = config
         self.key = key
@@ @@ -173,26 +136,27 @@ class Profile(object): @@
         self.actions = []
         for action in self._config_list(u'actions'):
             self.actions.append(self._config_action(action))
     def _config_action(self, name):
         """
         Retrieve an "action" value from config, for the current profile.  This
         returns a :class:`ProfileAction` instance.
         Retrieve an "action" value from config, for the current
         profile.  This returns a
         :class:`~rattail.config.ConfigProfileAction` instance.
         """
         function = self._config_string('action.{}.func'.format(name))
         class_ = self._config_string('action.{}.class'.format(name))
         cmd = self._config_string('action.{}.cmd'.format(name))
         specs = [1 if spec else 0 for spec in (function, class_, cmd)]
         if sum(specs) != 1:
             raise ConfigurationError(
                 "File monitor profile '{}' (action '{}') must have exactly one of: "
                 "function, class, command".format(self.key, name))
-        action = ProfileAction()
+        action = ConfigProfileAction()
         action.config = self.config
         if function:
             action.spec = function
             action.action = load_object(action.spec)
         elif class_:

rattail/mailmon/__init__.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8; -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
 #  Copyright © 2010-2021 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU General Public License as published by the Free Software
 #  Foundation, either version 3 of the License, or (at your option) any later
 #  version.
+#
 #  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 #  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 #  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 #  details.
+#
 #  You should have received a copy of the GNU General Public License along with
 #  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
 Mail Monitoring - more specifically, Email Folder Monitoring
 """
 from .actions import MessageAction

rattail/mailmon/actions.py

➞

Show inline comments

 # -*- coding: utf-8; -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
 #  Copyright © 2010-2021 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU General Public License as published by the Free Software
 #  Foundation, either version 3 of the License, or (at your option) any later
 #  version.
+#
 #  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 #  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 #  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 #  details.
+#
 #  You should have received a copy of the GNU General Public License along with
 #  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
 Mail Monitor Actions
 """
 from __future__ import unicode_literals, absolute_import
 import os
 import shutil
 import tempfile
 from rattail.monitoring import MonitorAction
 from rattail.config import parse_bool
 from rattail.files import locking_copy
 class MessageAction(MonitorAction):
     """
     Base class for mailmon message actions.
     """
     def __call__(self, server, msguid, *args, **kwargs):
         """
         This method must be implemented in the subclass; it defines
         what the action actually *does*.  The monitor daemon will
         invoke this method for all new messages which are discovered.
         :param server: Reference to the ``imaplib.server`` instance,
            which will be connected with an active session.
         :param msguid: UID for the message upon which to act.
         """
         raise NotImplementedError
 def download_message(server, msg_uid, output_dir, locking=False):
     """
     Simple action to "download" a message to local filesystem.
     :param output_dir: Path to the folder into which message should be
        written.  Note that the filename will be like
        ``{msg_uid}.eml``.
     :param locking: Flag to indicate that the
        :func:`rattail.files.locking_copy()` function should be used to
        place the file in its final location.  This is useful if you
        then also have a rattail filemon watching the ``output_dir``.
     """
     if not isinstance(locking, bool):
         locking = parse_bool(locking)
     # fetch message data
     code, msg_data = server.uid('fetch', msg_uid, '(RFC822)')
     if code != 'OK':
         raise RuntimeError("IMAP4.fetch() for msg_uid %s returned "
                            "bad code %s - msg_data is: %s",
                            msg_uid, code, msg_data)
     # extract message body
     # TODO: what do these signify?
     assert len(msg_data) == 2
     assert msg_data[1] == ')'
     response, msg_body = msg_data[0]
     # figure out where we need to write the file
     filename = '{}.eml'.format(msg_uid)
     if locking:
         tempdir = tempfile.mkdtemp()
         path = os.path.join(tempdir, filename)
     else: # no locking, write directly to file
         path = os.path.join(output_dir, filename)
     # write message to file
     with open(path, 'wb') as f:
         f.write(msg_body)
     # maybe move temp file to final path
     if locking:
         locking_copy(path, output_dir)
         shutil.rmtree(tempdir)
 def move_message(server, msguid, newfolder):
     """
     Simple action to "move" a message to another IMAP folder, on the
     same server.
     """
     # copy msg to new folder
     code, response = server.uid('COPY', msguid, newfolder)
     if code != 'OK':
         raise RuntimeError("IMAP.copy(uid={}) returned bad code: {}".format(
             msguid, code))
     # mark old msg as deleted
     code, response = server.uid('STORE', msguid, '+FLAGS', '(\Deleted)')
     if code != 'OK':
         raise RuntimeError("IMAP.store(uid={}) returned bad code: {}".format(
             msguid, code))
     # expunge deleted messages
     code, response = server.expunge()
     if code != 'OK':
         raise RuntimeError("IMAP.expunge() returned bad code: {}".format(code))

rattail/mailmon/config.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8; -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
 #  Copyright © 2010-2021 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU General Public License as published by the Free Software
 #  Foundation, either version 3 of the License, or (at your option) any later
 #  version.
+#
 #  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 #  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 #  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 #  details.
+#
 #  You should have received a copy of the GNU General Public License along with
 #  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
 Mail Monitor Configuration
 """
 from __future__ import unicode_literals, absolute_import
 import six
 from rattail.config import ConfigProfile, parse_list
 from rattail.exceptions import ConfigurationError
 class MailMonitorProfile(ConfigProfile):
     """
     Simple class to hold configuration for a MailMon "profile".  Each
     profile determines which email folder(s) will be watched for new
     messages, and which action(s) will then be invoked to process the
     messages.
     """
     section = 'rattail.mailmon'
     def load(self):
         self.imap_server = self._config_string('imap.server')
         self.imap_username = self._config_string('imap.username')
         self.imap_password = self._config_string('imap.password')
         self.imap_folder = self._config_string('imap.folder')
         self.imap_unread_only = self._config_boolean('imap.unread_only')
         self.imap_delay = self._config_int('imap.delay', default=120)
         self.imap_recycle = self._config_int('imap.recycle', default=1200)
         self.max_batch_size = self._config_int('max_batch_size', default=100)
         self.load_defaults()
         self.load_actions()
     def validate(self):
         """
         Validate the configuration for current profile.
         """
         if not self.actions:
             raise ConfigurationError("mailmon profile '{}' has no valid "
                                      "actions to invoke".format(self.key))
 def load_mailmon_profiles(config):
     """
     Load all active mail monitor profiles defined within configuration.
     """
     # make sure we have a top-level directive
     keys = config.get('rattail.mailmon', 'monitor')
     if not keys:
         raise ConfigurationError(
             "The mail monitor configuration does not specify any profiles "
             "to be monitored.  Please defined the 'monitor' option within "
             "the [rattail.mailmon] section of your config file.")
     monitored = {}
     for key in parse_list(keys):
         profile = MailMonitorProfile(config, key)
         # only monitor this profile if it validates
         try:
             profile.validate()
         except ConfigurationError as error:
             log.warning(six.text_type(error))
         else:
             monitored[key] = profile
     return monitored

rattail/mailmon/daemon.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8; -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
 #  Copyright © 2010-2021 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU General Public License as published by the Free Software
 #  Foundation, either version 3 of the License, or (at your option) any later
 #  version.
+#
 #  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 #  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 #  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 #  details.
+#
 #  You should have received a copy of the GNU General Public License along with
 #  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
 Mail Monitor Daemon
 """
 from __future__ import unicode_literals, absolute_import
 import re
 import time
 import imaplib
 from six.moves import queue
 # import sys
 import logging
 # from traceback import format_exception
 import six
 from rattail.daemon import Daemon
 from rattail.mailmon.config import load_mailmon_profiles
 from rattail.threads import Thread
 from rattail.time import make_utc
 from rattail.exceptions import StopProcessing
 # from rattail.mail import send_email
 log = logging.getLogger(__name__)
 class MailMonitorDaemon(Daemon):
     """
     Daemon responsible for checking IMAP folders and detecting email
     messages, and then invoking actions upon them.
     """
     def run(self):
         """
         Starts watcher and worker threads according to configuration.
         """
         monitored = load_mailmon_profiles(self.config)
         for key, profile in six.iteritems(monitored):
             # create a msg queue for the profile
             profile.queue = queue.Queue()
             # create a watcher thread for the IMAP folder
             watcher = IMAPWatcher(profile)
             name = 'watcher_{}'.format(key)
             log.info("starting IMAP watcher thread: %s", name)
             thread = Thread(target=watcher, name=name)
             thread.daemon = True
             thread.start()
             # create an action thread for the profile
             name = 'actions-{}'.format(key)
             log.debug("starting action thread: %s", name)
             thread = Thread(target=perform_actions, name=name,
                             args=(self.config, watcher))
             thread.daemon = True
             thread.start()
         # loop indefinitely.  since this is the main thread, the app
         # will terminate when this method ends; all other threads are
         # "subservient" to this one.
         while True:
             time.sleep(.01)
 class IMAPWatcher(object):
     """
     Abstraction to make watching an IMAP folder a little more
     organized.  Instances of this class are used as callable targets
     when the daemon starts watcher threads.  They are responsible for
     polling the IMAP folder and processing any messages found there.
     """
     uid_pattern = re.compile(r'^\d+ \(UID (?P<uid>\d+)')
     def __init__(self, profile):
         self.profile = profile
         self.server = None
     def get_uid(self, response):
         match = self.uid_pattern.match(response)
         if match:
             return match.group('uid')
     def __call__(self):
         recycled = None
         while True:
             if self.server is None:
                 self.server = imaplib.IMAP4_SSL(self.profile.imap_server)
                 try:
                     result = self.server.login(self.profile.imap_username, self.profile.imap_password)
                 except self.server.error:
                     log.exception("failed to login to server!")
                     return
                 log.debug("IMAP server login result: %s", result)
                 recycled = make_utc()
             self.server.select(self.profile.imap_folder)
             try:
                 self.queue_messages()
             except:
                 log.exception("failed to queue messages!")
                 if profile.stop_on_error:
                     break
             time.sleep(self.profile.imap_delay)
             # If recycle time limit has been reached, close and reopen the IMAP connection.
             if (make_utc() - recycled).seconds >= self.profile.imap_recycle:
                 log.debug("recycle time limit reached, disposing of current connection")
                 self.server.close()
                 self.server.logout()
                 self.server = None
         self.server.close()
         self.server.logout()
     def queue_messages(self):
         """
         Check for new messages in the folder, and queue any found, for
         action processing thread.
         """
         # maybe look for "all" or maybe just "unread"
         if self.profile.imap_unread_only:
             criterion = '(UNSEEN)'
         else:
             criterion = 'ALL'
         # log.debug("invoking IMAP4.search()")
         code, items = self.server.uid('search', None, criterion)
         if code != 'OK':
             raise RuntimeError("IMAP4.search() returned bad code: {}".format(code))
         # config may dictacte a "max batch size" in which case we will
         # only queue so many messages at a time
         uids = items[0].split()
         if self.profile.max_batch_size:
             if len(uids) > self.profile.max_batch_size:
                 uids = uids[:self.profile.max_batch_size]
         # add message uids to the queue
         for uid in uids:
             self.profile.queue.put(uid)
 def perform_actions(config, watcher):
     """
     Target for action threads.  Provides the main loop which checks
     the queue for new messages and invokes actions for each, as they
     appear.
     """
     profile = watcher.profile
     stop = False
     while not stop:
         # suspend execution briefly, to avoid consuming so much CPU...
         time.sleep(0.01)
         try:
             msguid = profile.queue.get_nowait()
         except queue.Empty:
             pass
         except StopProcessing:
             stop = True
         else:
             log.debug("queue contained a msguid: %s", msguid)
             for action in profile.actions:
                 try:
                     invoke_action(config, watcher, action, msguid)
                 except:
                     # stop processing messages altogether for this
                     # profile if it is so configured
                     if profile.stop_on_error:
                         log.warning("an error was encountered, and config "
                                     "dictates that no more actions should be "
                                     "processed for profile: %s", profile.key)
                         stop = True
                     # either way no more actions should be invoked for
                     # this particular message
                     break
 def invoke_action(config, watcher, action, msguid):
     """
     Invoke a single action on a mail message, retrying as necessary.
     """
     attempts = 0
     errtype = None
     while True:
         attempts += 1
         log.debug("invoking action '%s' (attempt #%s of %s) on file: %s",
                   action.spec, attempts, action.retry_attempts, msguid)
         try:
             action.action(watcher.server, msguid, *action.args, **action.kwargs)
         except:
             # if we've reached our final attempt, stop retrying
             if attempts >= action.retry_attempts:
                 # log.debug("attempt #%s failed for action '%s' (giving up) on "
                 #           "msguid: %s", attempts, action.spec, msguid,
                 #           exc_info=True)
                 log.exception("attempt #%s failed for action '%s' (giving up) on "
                           "msguid: %s", attempts, action.spec, msguid)
                 # TODO: add email support
                 # exc_type, exc, traceback = sys.exc_info()
                 # send_email(config, 'mailmon_action_error', {
                 #     # 'hostname': socket.gethostname(),
                 #     # 'path': path,
                 #     'msguid': msguid,
                 #     'action': action,
                 #     'attempts': attempts,
                 #     'error': exc,
                 #     'traceback': ''.join(format_exception(exc_type, exc, traceback)).strip(),
                 # })
                 raise
             # if this exception is not the first, and is of a
             # different type than seen previously, do *not* continue
             # to retry
             if errtype is not None and not isinstance(error, errtype):
                 log.exception("new exception differs from previous one(s), "
                               "giving up on action '%s' for msguid: %s",
                               action.spec, msguid)
                 raise
             # record the type of exception seen, and pause for next retry
             log.warning("attempt #%s failed for action '%s' on msguid: %s",
                         attempts, action.spec, msguid, exc_info=True)
             errtype = type(error)
             log.debug("pausing for %s seconds before making attempt #%s of %s",
                       action.retry_delay, attempts + 1, action.retry_attempts)
             if action.retry_delay:
                 time.sleep(action.retry_delay)
         else:
             # no error, invocation successful
             log.debug("attempt #%s succeeded for action '%s' on msguid: %s",
                       attempts, action.spec, msguid)
             break

rattail/monitoring.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8; -*-
 ################################################################################
+#
 #  Rattail -- Retail Software Framework
 #  Copyright © 2010-2021 Lance Edgar
+#
 #  This file is part of Rattail.
+#
 #  Rattail is free software: you can redistribute it and/or modify it under the
 #  terms of the GNU General Public License as published by the Free Software
 #  Foundation, either version 3 of the License, or (at your option) any later
 #  version.
+#
 #  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
 #  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 #  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 #  details.
+#
 #  You should have received a copy of the GNU General Public License along with
 #  Rattail.  If not, see <http://www.gnu.org/licenses/>.
+#
 ################################################################################
 """
 Monitoring Library
 This contains misc. common/shared logic for use with multiple types of
 monitors, e.g. datasync, filemon etc.
 """
 from __future__ import unicode_literals, absolute_import
 import os
 import logging
 import subprocess
 from rattail.config import parse_bool, parse_list
 log = logging.getLogger(__name__)
 class MonitorAction(object):
     """
     Base class for monitor actions.  Note that not all actions are
     class-based, but the ones which are should probably inherit from
     this class.
     """
     def __init__(self, config):
         self.config = config
         self.app = config.get_app()
         self.enum = config.get_enum()
         self.model = config.get_model()
     def __call__(self, *args, **kwargs):
         """
         This method must be implemented in the subclass; it defines
         what the action actually *does*.  The monitor daemon will
         invoke this method for all new items which are discovered.
         """
         raise NotImplementedError
 class CommandAction(MonitorAction):
     """
     Simple action which can execute an arbitrary command, as a
     subprocess.  This action is meant to be invoked with a particular
     file path, which is to be acted upon.
     """
     def __init__(self, config, cmd):
         super(CommandAction, self).__init__(config)
         self.cmd = cmd
     def __call__(self, path, **kwargs):
         """
         Run the command for the given file path.
         """
         filename = os.path.basename(path)
         shell = parse_bool(kwargs.pop('shell', False))
         if shell:
             # TODO: probably shoudn't use format() b/c who knows what is in
             # that command line, that might trigger errors
             cmd = self.cmd.format(path=path, filename=filename)
         else:
             cmd = []
             for term in parse_list(self.cmd):
                 term = term.replace('{path}', path)
                 term = term.replace('{filename}', filename)
                 cmd.append(term)
         log.debug("final command to run is: %s", cmd)
         subprocess.check_call(cmd, shell=shell)

rattail/tests/filemon/test_actions.py

➞

Show inline comments

 import tempfile
 from six.moves import queue
 from unittest import TestCase
 from mock import Mock, patch, call
 from rattail.config import make_config, RattailConfig
+from rattail.config import make_config, RattailConfig, ConfigProfileAction
 from rattail.filemon import actions
-from rattail.filemon.config import Profile, ProfileAction
 from rattail.filemon.config import Profile
 class TestAction(TestCase):
     def test_callable_must_be_implemented_in_subclass(self):
         config = make_config([])
         self.assertTrue(os.path.exists(os.path.join(self.tempdir, 'file3')))
 class TestInvokeAction(TestCase):
     def setUp(self):
-        self.action = ProfileAction()
+        self.action = ConfigProfileAction()
         self.action.config = RattailConfig()
         self.action.action = Mock(return_value=None)
         self.action.retry_attempts = 6
         self.tempdir = tempfile.mkdtemp()
         self.file = self.write_file('file', '')

rattail/tests/test_monitoring.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8; -*-
 from __future__ import unicode_literals, absolute_import
 import unittest
 from mock import patch, Mock
 from rattail import monitoring
 from rattail.config import RattailConfig
 class TestMonitorAction(unittest.TestCase):
     def setUp(self):
         self.config = RattailConfig()
     def test_attributes(self):
         action = monitoring.MonitorAction(self.config)
         self.assertIs(action.config, self.config)
         self.assertTrue(hasattr(action, 'app'))
     def test_not_implemented(self):
         action = monitoring.MonitorAction(self.config)
         self.assertRaises(NotImplementedError, action)
 class TestCommandAction(unittest.TestCase):
     def setUp(self):
         self.config = RattailConfig()
     def test_attributes(self):
         action = monitoring.CommandAction(self.config, "echo test")
         self.assertIs(action.config, self.config)
         self.assertTrue(hasattr(action, 'app'))
         self.assertEqual(action.cmd, "echo test")
     @patch('rattail.monitoring.subprocess')
     def test_run_invokes_command(self, subprocess):
         subprocess.check_call = Mock()
         action = monitoring.CommandAction(self.config, "echo {filename}")
         action('test.txt')
         self.assertEqual(subprocess.check_call.call_count, 1)
         # nb. shell=False is a default kwarg
         subprocess.check_call.assert_called_with(['echo', 'test.txt'], shell=False)
     @patch('rattail.monitoring.subprocess')
     def test_run_with_shell(self, subprocess):
         subprocess.check_call = Mock()
         action = monitoring.CommandAction(self.config, "echo {filename}")
         action('test.txt', shell=True)
         self.assertEqual(subprocess.check_call.call_count, 1)
         subprocess.check_call.assert_called_with('echo test.txt', shell=True)

setup.py

➞

Show inline comments

@@ @@ -208,12 +208,13 @@ export-rattail = rattail.commands.importing:ExportRattail @@
 filemon = rattail.commands.core:FileMonitorCommand
 import-csv = rattail.commands.importing:ImportCSV
 import-ifps = rattail.commands.importing:ImportIFPS
 import-rattail = rattail.commands.importing:ImportRattail
 import-sample = rattail.commands.importing:ImportSampleData
 import-versions = rattail.commands.importing:ImportVersions
 mailmon = rattail.commands.core:MailMonitorCommand
 make-appdir = rattail.commands.core:MakeAppDir
 make-batch = rattail.commands.batch:MakeBatch
 make-config = rattail.commands.core:MakeConfig
 make-user = rattail.commands.core:MakeUser
 make-uuid = rattail.commands.core:MakeUUID
 populate-batch = rattail.commands.batch:PopulateBatch

0 comments (0 inline, 0 general)