From 2c7d6c16e14f26caac605ea5215b0977eaadbd0a 2014-07-04 14:41:28 From: Lance Edgar Date: 2014-07-04 14:41:28 Subject: [PATCH] File Monitor overhaul! * New configuration syntax (old syntax still supported but deprecated). * Class-based actions. * Configure keyword arguments to action callables. * Configure retry for actions. * Add (some) tests, docs. --- diff --git a/.gitignore b/.gitignore index e87630d909a6d3401a5943ecc83c0578bf416b49..c0c5008cfe6a15bcbae010550d098e2d14a3e07f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .tox/ +docs/_build/ rattail.egg-info/ diff --git a/docs/api/rattail/filemon/actions.rst b/docs/api/rattail/filemon/actions.rst new file mode 100644 index 0000000000000000000000000000000000000000..d5e248864783ce5c089394eecab42c31e93e6cd0 --- /dev/null +++ b/docs/api/rattail/filemon/actions.rst @@ -0,0 +1,7 @@ +.. -*- coding: utf-8 -*- + +``rattail.filemon.actions`` +=========================== + +.. automodule:: rattail.filemon.actions + :members: diff --git a/docs/api/rattail/filemon/config.rst b/docs/api/rattail/filemon/config.rst new file mode 100644 index 0000000000000000000000000000000000000000..993da0103c3dfff1b0ba371b08464c3f1f281b85 --- /dev/null +++ b/docs/api/rattail/filemon/config.rst @@ -0,0 +1,7 @@ +.. -*- coding: utf-8 -*- + +``rattail.filemon.config`` +========================== + +.. automodule:: rattail.filemon.config + :members: diff --git a/docs/api/rattail/filemon/index.rst b/docs/api/rattail/filemon/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..0ccaa85e0c25150ca39fc24f882f243a1d48b310 --- /dev/null +++ b/docs/api/rattail/filemon/index.rst @@ -0,0 +1,8 @@ +.. -*- coding: utf-8 -*- + +``rattail.filemon`` +=================== + +.. automodule:: rattail.filemon + +.. autoclass:: Action diff --git a/docs/api/rattail/filemon/linux.rst b/docs/api/rattail/filemon/linux.rst new file mode 100644 index 0000000000000000000000000000000000000000..832458a53eb5035fd5ab6b58f29cf5930b11a541 --- /dev/null +++ b/docs/api/rattail/filemon/linux.rst @@ -0,0 +1,7 @@ +.. -*- coding: utf-8 -*- + +``rattail.filemon.linux`` +========================= + +.. automodule:: rattail.filemon.linux + :members: diff --git a/docs/api/rattail/filemon/util.rst b/docs/api/rattail/filemon/util.rst index 64ae9206b221591c640b07905d51ea3e4c19eaf2..9c04520eac57bf07abf5b2d940d096dd69ddaca5 100644 --- a/docs/api/rattail/filemon/util.rst +++ b/docs/api/rattail/filemon/util.rst @@ -1,3 +1,4 @@ +.. -*- coding: utf-8 -*- ``rattail.filemon.util`` ======================== diff --git a/docs/api/rattail/filemon/win32.rst b/docs/api/rattail/filemon/win32.rst new file mode 100644 index 0000000000000000000000000000000000000000..9ed13618110a8b30b3f765b9d117ea5d86016570 --- /dev/null +++ b/docs/api/rattail/filemon/win32.rst @@ -0,0 +1,7 @@ +.. -*- coding: utf-8 -*- + +``rattail.filemon.win32`` +========================= + +.. automodule:: rattail.filemon.win32 + :members: diff --git a/docs/index.rst b/docs/index.rst index c60c6004603a822f8a70c7daae118e490637a0f5..83f49523667724b27a20f2a19678e71af5bfd4a3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,7 +20,12 @@ Package API: api/rattail/db/util api/rattail/enum api/rattail/exceptions + api/rattail/filemon/index + api/rattail/filemon/actions + api/rattail/filemon/config + api/rattail/filemon/linux api/rattail/filemon/util + api/rattail/filemon/win32 api/rattail/logging diff --git a/rattail/exceptions.py b/rattail/exceptions.py index d6b1dd559cb1f0e5b17485b62dc61bc99534a8ea..e27dfa3880e66d1f8462d733525cff326ac4a6a4 100644 --- a/rattail/exceptions.py +++ b/rattail/exceptions.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2012 Lance Edgar +# Copyright © 2010-2014 Lance Edgar # # This file is part of Rattail. # @@ -35,6 +34,12 @@ class RattailError(Exception): """ +class ConfigurationError(RattailError): + """ + Generic class for configuration errors. + """ + + class BatchError(RattailError): """ Base class for all batch-related errors. diff --git a/rattail/filemon/__init__.py b/rattail/filemon/__init__.py index c4456bfb90aaa1cb2ba4a0bdffe3443d1f28e04a..58c11c9b76121ceddfcd494e4ac061477e1259cf 100644 --- a/rattail/filemon/__init__.py +++ b/rattail/filemon/__init__.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2012 Lance Edgar +# Copyright © 2010-2014 Lance Edgar # # This file is part of Rattail. # @@ -23,204 +22,7 @@ ################################################################################ """ -``rattail.filemon`` -- File Monitor +File Monitor """ -import os -import os.path -import sys -import Queue -import logging - -from ..util import load_object - -if sys.platform == 'win32': - import win32api - from rattail.win32 import file_is_free - - -log = logging.getLogger(__name__) - - -class MonitorProfile(object): - """ - This is a simple profile class, used to represent configuration of the file - monitor service. - """ - - def __init__(self, config, key): - self.config = config - self.key = key - - self.dirs = config.require('rattail.filemon', '{0}.dirs'.format(key)) - self.dirs = eval(self.dirs) - - actions = config.require('rattail.filemon', '{0}.actions'.format(key)) - actions = eval(actions) - - self.actions = [] - for action in actions: - if isinstance(action, tuple): - spec = action[0] - args = list(action[1:]) - else: - spec = action - args = [] - func = load_object(spec) - self.actions.append((spec, func, args)) - - self.locks = config.getboolean( - 'rattail.filemon', '{0}.locks'.format(key), default=False) - - self.process_existing = config.getboolean( - 'rattail.filemon', '{0}.process_existing'.format(key), default=True) - - self.stop_on_error = config.getboolean( - 'rattail.filemon', '{0}.stop_on_error'.format(key), default=False) - - -def get_monitor_profiles(config): - """ - Convenience function to load monitor profiles from config. - """ - - monitored = {} - - # Read monitor profile(s) from config. - keys = config.require('rattail.filemon', 'monitored') - keys = keys.split(',') - for key in keys: - key = key.strip() - log.debug("get_monitor_profiles: loading profile: {0}".format(key)) - profile = MonitorProfile(config, key) - monitored[key] = profile - for path in profile.dirs[:]: - - # Ensure the monitored path exists. - if not os.path.exists(path): - log.warning("get_monitor_profiles: profile '{0}' has nonexistent " - "path, which will be pruned: {1}".format(key, path)) - profile.dirs.remove(path) - - # Ensure the monitored path is a folder. - elif not os.path.isdir(path): - log.warning("get_monitor_profiles: profile '{0}' has non-folder " - "path, which will be pruned: {1}".format(key, path)) - profile.dirs.remove(path) - - for key in monitored.keys(): - profile = monitored[key] - - # Prune any profiles with no valid folders to monitor. - if not profile.dirs: - log.warning("get_monitor_profiles: profile has no folders to " - "monitor, and will be pruned: {0}".format(key)) - del monitored[key] - - # Prune any profiles with no valid actions to perform. - elif not profile.actions: - log.warning("get_monitor_profiles: profile has no actions to " - "perform, and will be pruned: {0}".format(key)) - del monitored[key] - - return monitored - - -def queue_existing(profile, path): - """ - Adds files found in a watched folder to a processing queue. This is called - when the monitor first starts, to handle the case of files which exist - prior to startup. - - If files are found, they are first sorted by modification timestamp, using - a lexical sort on the filename as a tie-breaker, and then added to the - queue in that order. - - :param profile: Monitor profile for which the folder is to be watched. The - profile is expected to already have a queue attached; any existing files - will be added to this queue. - :type profile: :class:`rattail.filemon.MonitorProfile` instance - - :param path: Folder path which is to be checked for files. - :type path: string - - :returns: ``None`` - """ - - def sorter(x, y): - mtime_x = os.path.getmtime(x) - mtime_y = os.path.getmtime(y) - if mtime_x < mtime_y: - return -1 - if mtime_x > mtime_y: - return 1 - return cmp(x, y) - - paths = [os.path.join(path, x) for x in os.listdir(path)] - for path in sorted(paths, cmp=sorter): - - # Only process normal files. - if not os.path.isfile(path): - continue - - # If using locks, don't process "in transit" files. - if profile.locks and path.endswith('.lock'): - continue - - log.debug("queue_existing: queuing existing file for " - "profile '{0}': {1}".format(profile.key, path)) - profile.queue.put(path) - - -def perform_actions(profile): - """ - Callable target for action threads. - """ - - keep_going = True - while keep_going: - - try: - path = profile.queue.get_nowait() - except Queue.Empty: - pass - else: - - # In some cases, processing one file may cause other related files - # to also be processed. When this happens, a path on the queue may - # point to a file which no longer exists. - if not os.path.exists(path): - log.info("perform_actions: path does not exist: {0}".format(path)) - continue - - log.debug("perform_actions: processing file: {0}".format(path)) - - if sys.platform == 'win32': - while not file_is_free(path): - win32api.Sleep(0) - - for spec, func, args in profile.actions: - - log.info("perform_actions: calling function '{0}' on file: {1}".format( - spec, path)) - - try: - func(path, *args) - - except: - log.exception(u"error processing file: %s", path) - - # Don't process any more files if the profile is so - # configured. - if profile.stop_on_error: - keep_going = False - - # Either way this particular file probably shouldn't be - # processed any further. - log.warning("perform_actions: no further processing will " - "be done for file: {0}".format(path)) - break - - log.warning("perform_actions: error encountered, and configuration " - "dictates that no more actions will be processed for " - "profile: {0}".format(profile.key)) +from rattail.filemon.actions import Action diff --git a/rattail/filemon/actions.py b/rattail/filemon/actions.py new file mode 100644 index 0000000000000000000000000000000000000000..a7db188f61bbdc8f2b5f76a2a4b9b5b365ac46e1 --- /dev/null +++ b/rattail/filemon/actions.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2014 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Rattail. If not, see . +# +################################################################################ + +""" +File Monitor Actions +""" + +import os +import sys +import time +import Queue +import logging + + +log = logging.getLogger(__name__) + + +class Action(object): + """ + Base class for file monitor actions. + """ + + def __init__(self, config): + self.config = config + + def __call__(self, *args, **kwargs): + """ + This method must be implemented in the subclass; it defines what the + action actually *does*. The file monitor will invoke this method for + all new files which are discovered. + """ + raise NotImplementedError + + +class StopProcessing(Exception): + """ + Simple exception to indicate action processing should stop. This is really + only useful for tests. + """ + + +def perform_actions(profile): + """ + Target for action threads. Provides the main loop which checks the queue + for new files and invokes actions for each, as they appear. + """ + + # If running on Windows, we add a step to help ensure the file is truly + # free of competing process interests. (In fact it would be nice to have + # this on Linux as well, but I'm not sure how to do it there.) + wait_for_file = lambda p: p + if sys.platform.startswith(u'win'): # pragma: no cover + import win32api + from rattail.win32 import file_is_free + def wait_for_file(path): + while not file_is_free(path): + win32api.Sleep(0) + + stop = False + while not stop: + + try: + path = profile.queue.get_nowait() + except Queue.Empty: + pass + except StopProcessing: + stop = True + else: + log.debug(u"queue contained a file: {0}".format(repr(path))) + + # In some cases, processing one file may cause other related files + # to also be processed. When this happens, a path on the queue may + # point to a file which no longer exists. + if not os.path.exists(path): + log.warning(u"file path does not exist: {0}".format(path)) + continue + + # This does nothing unless running on Windows. + wait_for_file(path) + + for action in profile.actions: + try: + invoke_action(action, path) + + except: + # Stop processing files altogether for this profile if it + # is so configured. + if profile.stop_on_error: + log.warning(u"an error was encountered, and configuration dictates that no more " + u"actions will be processed for profile {0}".format(repr(profile.key))) + stop = True + + # Either way no more actions should be invoked for this + # particular file. + break + + +def invoke_action(action, path): + """ + Invoke a single action on a file, retrying as necessary. + """ + attempts = 0 + errtype = None + while True: + attempts += 1 + log.debug(u"invoking action {0} (attempt #{1} of {2}) on file: {3}".format( + repr(action.spec), attempts, action.retry_attempts, repr(path))) + + try: + action.action(path, *action.args, **action.kwargs) + + except Exception as error: + + # If we've reached our final attempt, stop retrying. + if attempts >= action.retry_attempts: + log.exception(u"attempt #{0} failed for action {1} (giving up) on " + u"file: {2}".format(attempts, repr(action.spec), repr(path))) + raise + + # If this exception is not the first, and is of a different type + # than seen previously, do *not* continue to retry. + if errtype is not None and not isinstance(error, errtype): + log.exception(u"new exception differs from previous one(s), giving up on " + u"action {0} for file: {1}".format(repr(action.spec), repr(path))) + raise + + # Record the type of exception seen, and pause for next retry. + log.warning(u"attempt #{0} failed for action {1} on file: {2}".format( + attempts, repr(action.spec), repr(path)), exc_info=True) + errtype = type(error) + log.debug(u"pausing for {0} seconds before making attempt #{1} of {2}".format( + action.retry_delay, attempts + 1, action.retry_attempts)) + if action.retry_delay: + time.sleep(action.retry_delay) + + else: + # No error, invocation successful. + log.debug(u"attempt #{0} succeeded for action {1} on file: {2}".format( + attempts, repr(action.spec), repr(path))) + break + + +def raise_exception(path, message=u"Fake error for testing"): + """ + File monitor action which always raises an exception. + + This is meant to be a simple way to test the error handling of a file + monitor. For example, whether or not file processing continues for + subsequent files after the first error is encountered. If logging + configuration dictates that an email should be sent, it will of course test + that as well. + """ + raise Exception(u'{0}: {1}'.format(message, path)) + + +def noop(path): + """ + File monitor action which does nothing at all. + + This exists for the sake of tests. I doubt it's useful in any other + context. + """ diff --git a/rattail/filemon/config.py b/rattail/filemon/config.py new file mode 100644 index 0000000000000000000000000000000000000000..0cee9888e588dfc6e70784d109feb24b17e0f577 --- /dev/null +++ b/rattail/filemon/config.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2014 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Rattail. If not, see . +# +################################################################################ + +""" +File Monitor Configuration +""" + +import os +import re +import shlex +import warnings +import logging + +from rattail.util import load_object +from rattail.exceptions import ConfigurationError + + +log = logging.getLogger(__name__) + + +class ProfileAction(object): + """ + Simple class to hold configuration for a particular action defined within a + monitor :class:`Profile`. Each instance has the following attributes: + + .. attribute:: spec + + The original "spec" string used to obtain the action callable. + + .. attribute:: action + + A reference to the action callable. + + .. attribute:: args + + A sequence of positional arguments to be passed to the callable (in + addition to the file path) when invoking the action. + + .. attribute:: kwargs + + A dictionary of keyword arguments to be passed to the callable (in + addition to the positional arguments) when invoking the action. + + .. attribute:: retry_attempts + + Number of attempts to make when invoking the action. Defaults to ``1``, + meaning the first attempt will be made but no retries will happen. + + .. attribute:: retry_delay + + Number of seconds to pause between retry attempts, if + :attr:`retry_attempts` is greater than one. Defaults to ``0``. + """ + + spec = None + action = None + args = [] + kwargs = {} + retry_attempts = 1 + retry_delay = 0 + + +class Profile(object): + """ + Simple class to hold configuration for a file monitor "profile". The + profile determines which folders to watch, which actions to take when new + files appear, etc. Each instance of this class has the following + attributes: + + .. attribute:: config + + Reference to the underlying configuration object from which the profile + derives its other attributes. + + .. attribute:: key + + String which differentiates this profile from any others which may exist + within the configuration. + + .. attribute:: dirs + + List of directory paths which should be watched by the file monitor on + behalf of this profile. + + .. attribute:: watch_locks + + Whether the file monitor should watch for new files, or disappearance of + file locks. + + If ``False`` (the default), the file monitor will not use any special + logic and will simply be on the lookout for any new files to appear; + when they do, the configured action(s) will be invoked. + + If this setting is ``True``, then the simple appearance of a new file + will not in itself cause any action(s) being taken. Instead, the file + monitor will only watch for the *disappearance* of file "locks", + i.e. any file or directory whose name ends in ``".lock"``. When the + disappearance of such a lock is noticed, the configured action(s) will + be invoked. + + .. attribute:: process_existing + + Whether any pre-existing files present in the watched folder(s) should + be automatically added to the processing queue. + + If this is ``True`` (the default), then when the file monitor first + starts, it will examine each watched folder to see if it already + contains files. If it does, then each file will be added to the + processing queue exactly as if the monitor had just noticed the file + appear. The files will be added in order of creation timestam, in an + effort to be consistent with the real-time behavior. + + If this setting is ``False``, any files which happen to exist when the + file monitor first starts will be ignored. + + .. attribute:: stop_on_error + + Whether the file monitor should halt processing for all future files, if + a single error is encountered. + + If this is ``True``, then if a single file action raises an exception, + all subsequent files to appear will be effectively ignored. No further + actions will be taken for any of them. + + If this is ``False`` (the default), then a single file error will only + cause processing to halt for that particular file. Subsequent files + which appear will still be processed as normal. + + .. attribute:: actions + + List of :class:`ProfileAction` instances representing the actions to be + invoked when new files are discovered. + """ + + def __init__(self, config, key): + self.config = config + self.key = key + + self.dirs = self.normalize_dirs(self._config_list(u'dirs')) + self.watch_locks = self._config_boolean(u'watch_locks', False) + self.process_existing = self._config_boolean(u'process_existing', True) + self.stop_on_error = self._config_boolean(u'stop_on_error', False) + + self.actions = [] + for action in self._config_list(u'actions'): + self.actions.append(self._config_action(action)) + + def _config_action(self, name): + function = self._config_string(u'action.{0}.func'.format(name)) + class_ = self._config_string(u'action.{0}.class'.format(name)) + + if function and class_: + raise ConfigurationError( + u"File monitor profile '{0}' has both function *and* class defined for " + u"action '{1}' (must have one or the other).".format(self.key, name)) + + if not function and not class_: + raise ConfigurationError( + u"File monitor profile '{0}' has neither function *nor* class defined for " + u"action '{1}' (must have one or the other).".format(self.key, name)) + + action = ProfileAction() + + if function: + action.spec = function + action.action = load_object(action.spec) + else: + action.spec = class_ + action.action = load_object(action.spec)(self.config) + + action.args = self._config_list(u'action.{0}.args'.format(name)) + + action.kwargs = {} + pattern = re.compile(ur'^{0}\.action\.{1}\.kwarg\.(?P\w+)$'.format(self.key, name), re.IGNORECASE) + for option in self.config.options(u'rattail.filemon'): + match = pattern.match(option) + if match: + action.kwargs[match.group(u'keyword')] = self.config.get(u'rattail.filemon', option) + + action.retry_attempts = self._config_int(u'action.{0}.retry_attempts'.format(name), minimum=1) + action.retry_delay = self._config_int(u'action.{0}.retry_delay'.format(name), minimum=0) + return action + + def _config_boolean(self, option, default=None): + return self.config.getboolean( + u'rattail.filemon', u'{0}.{1}'.format(self.key, option), default=default) + + def _config_int(self, option, minimum=0): + option = u'{0}.{1}'.format(self.key, option) + if self.config.has_option(u'rattail.filemon', option): + value = self.config.getint(u'rattail.filemon', option) + if value < minimum: + log.warning(u"config value {0} is too small; falling back to minimum " + u"of {1} for option: {2}".format(value, minimum, option)) + value = minimum + else: + value = minimum + return value + + def _config_list(self, option): + return parse_list(self._config_string(option)) + + def _config_string(self, option): + return self.config.get(u'rattail.filemon', u'{0}.{1}'.format(self.key, option)) + + def normalize_dirs(self, dirs): + """ + Normalize a list of directory paths. Converts all to absolute paths, + and prunes those which do not exist or do not refer to directories. + """ + normalized = [] + for path in dirs: + path = os.path.abspath(path) + if not os.path.exists(path): + log.warning(u"pruning folder which does not exist for profile " + u"{0}: {1}".format(repr(self.key), repr(path))) + continue + if not os.path.isdir(path): + log.warning(u"pruning folder which is not a directory for profile " + u"{0}: {1}".format(repr(self.key), repr(path))) + continue + normalized.append(path) + return normalized + + +def load_profiles(config): + """ + Load all active file monitor profiles defined within configuration. + """ + # Make sure we have a top-level directive. + keys = config.get(u'rattail.filemon', u'monitor') + if not keys: + + # No? What about with the old syntax? + keys = config.get(u'rattail.filemon', u'monitored') + if keys: + return load_legacy_profiles(config) + + # Still no? That's no good.. + raise ConfigurationError( + u"The file monitor configuration does not specify any profiles " + u"to be monitored. Please defined the 'monitor' option within " + u"the [rattail.filemon] section of your config file.") + + monitored = {} + for key in parse_list(keys): + profile = Profile(config, key) + if not profile.dirs: + log.warning(u"profile '{0}' has no valid directories to watch".format(key)) + continue + if not profile.actions: + log.warning(u"profile '{0}' has no valid actions to invoke".format(key)) + continue + monitored[key] = profile + + return monitored + + +class LegacyProfile(Profile): + """ + This is a file monitor profile corresponding to the first generation of + configuration syntax. E.g. it only supports function-based actions, and + does not support keyword arguments. + + .. note:: + This profile type will be deprecated at some point in the future. All + new code should assume use :class:`Profile` instead. + """ + + def __init__(self, config, key): + self.config = config + self.key = key + + dirs = self._config_string(u'dirs') + if dirs: + self.dirs = eval(dirs) + else: + self.dirs = [] + self.dirs = self.normalize_dirs(self.dirs) + + actions = self._config_string(u'actions') + if actions: + actions = eval(actions) + else: + actions = [] + + self.actions = [] + for action in actions: + if isinstance(action, tuple): + spec = action[0] + args = list(action[1:]) + else: + spec = action + args = [] + action = load_object(spec) + self.actions.append((spec, action, args, {})) + + self.watch_locks = self._config_boolean(u'locks', False) + self.process_existing = self._config_boolean(u'process_existing', True) + self.stop_on_error = self._config_boolean(u'stop_on_error', False) + + +def load_legacy_profiles(config): + """ + Load all active *legacy* file monitor profiles defined within + configuration. + """ + warnings.warn(u"The use of legacy profiles is deprecated, and should be " + u"avoided. Please update your configuration to use the new " + u"syntax as soon as possible.", DeprecationWarning) + + monitored = {} + + # Read monitor profile(s) from config. + keys = config.get(u'rattail.filemon', u'monitored') + if not keys: + raise ConfigurationError( + u"The file monitor configuration does not specify any profiles " + u"to be monitored. Please defined the 'monitor' option within " + u"the [rattail.filemon] section of your config file.") + keys = keys.split(u',') + for key in keys: + key = key.strip() + log.debug(u"loading profile: {0}".format(key)) + monitored[key] = LegacyProfile(config, key) + + for key in monitored.keys(): + profile = monitored[key] + + # Prune any profiles with no valid folders to monitor. + if not profile.dirs: + log.warning(u"profile has no folders to monitor, " + u"and will be pruned: {0}".format(key)) + del monitored[key] + + # Prune any profiles with no valid actions to perform. + elif not profile.actions: + log.warning(u"profile has no actions to perform, " + u"and will be pruned: {0}".format(key)) + del monitored[key] + + return monitored + + +def parse_list(value): + """ + Parse a configuration value, splitting by whitespace and/or commas and + taking quoting into account, etc., yielding a list of strings. + """ + if value is None: + return [] + parser = shlex.shlex(value) + parser.whitespace += u',' + parser.whitespace_split = True + values = list(parser) + for i, value in enumerate(values): + if value.startswith(u'"') and value.endswith(u'"'): + values[i] = value[1:-1] + return values diff --git a/rattail/filemon/linux.py b/rattail/filemon/linux.py index b71e35e73b42974f7ab34b880dfa99d631418f12..7d9100b7a6816aac2b5713915678b86192a6801b 100644 --- a/rattail/filemon/linux.py +++ b/rattail/filemon/linux.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2012 Lance Edgar +# Copyright © 2010-2014 Lance Edgar # # This file is part of Rattail. # @@ -23,29 +22,19 @@ ################################################################################ """ -``rattail.filemon.linux`` -- File Monitor for Linux +File Monitor for Linux """ -import sys -import os.path -import threading import Queue import logging -try: - import pyinotify -except ImportError: - # Mock out for Windows. - class Dummy(object): - pass - pyinotify = Dummy() - pyinotify.ProcessEvent = Dummy - -import edbob -from edbob.errors import email_exception +import pyinotify from rattail.daemon import Daemon -from rattail import filemon +from rattail.threads import Thread +from rattail.filemon.config import load_profiles +from rattail.filemon.actions import perform_actions +from rattail.filemon.util import queue_existing log = logging.getLogger(__name__) @@ -53,46 +42,50 @@ log = logging.getLogger(__name__) class EventHandler(pyinotify.ProcessEvent): """ - Event processor for file monitor daemon. + Event processor for file monitor daemon. This receives notifications of + file system events, and places new files on the queue as appropriate. """ def my_init(self, profile=None, **kwargs): self.profile = profile def process_IN_ACCESS(self, event): - log.debug("EventHandler: IN_ACCESS: %s" % event.pathname) + log.debug(u"IN_ACCESS: {0}".format(event.pathname)) def process_IN_ATTRIB(self, event): - log.debug("EventHandler: IN_ATTRIB: %s" % event.pathname) + log.debug(u"IN_ATTRIB: {0}".format(event.pathname)) def process_IN_CLOSE_WRITE(self, event): - log.debug("EventHandler: IN_CLOSE_WRITE: %s" % event.pathname) - if not self.profile.locks: + log.debug(u"IN_CLOSE_WRITE: {0}".format(event.pathname)) + if not self.profile.watch_locks: self.profile.queue.put(event.pathname) def process_IN_CREATE(self, event): - log.debug("EventHandler: IN_CREATE: %s" % event.pathname) + log.debug(u"IN_CREATE: {0}".format(event.pathname)) def process_IN_DELETE(self, event): - log.debug("EventHandler: IN_DELETE: %s" % event.pathname) - if self.profile.locks and event.pathname.endswith('.lock'): + log.debug(u"IN_DELETE: {0}".format(event.pathname)) + if self.profile.watch_locks and event.pathname.endswith(u'.lock'): self.profile.queue.put(event.pathname[:-5]) def process_IN_MODIFY(self, event): - log.debug("EventHandler: IN_MODIFY: %s" % event.pathname) + log.debug(u"IN_MODIFY: {0}".format(event.pathname)) def process_IN_MOVED_TO(self, event): - log.debug("EventHandler: IN_MOVED_TO: %s" % event.pathname) - if not self.profile.locks: + log.debug(u"IN_MOVED_TO: {0}".format(event.pathname)) + if not self.profile.watch_locks: self.profile.queue.put(event.pathname) class FileMonitorDaemon(Daemon): + """ + Linux daemon implementation of the File Monitor. + """ def run(self): - wm = pyinotify.WatchManager() - notifier = pyinotify.Notifier(wm) + watch_manager = pyinotify.WatchManager() + notifier = pyinotify.Notifier(watch_manager) mask = (pyinotify.IN_ACCESS | pyinotify.IN_ATTRIB @@ -102,7 +95,7 @@ class FileMonitorDaemon(Daemon): | pyinotify.IN_MODIFY | pyinotify.IN_MOVED_TO) - monitored = filemon.get_monitor_profiles(self.config) + monitored = load_profiles(self.config) for key, profile in monitored.iteritems(): # Create a file queue for the profile. @@ -113,17 +106,16 @@ class FileMonitorDaemon(Daemon): # Maybe put all pre-existing files in the queue. if profile.process_existing: - filemon.queue_existing(profile, path) + queue_existing(profile, path) # Create a watch for the folder. - log.debug("start_daemon: profile '%s' watches folder: %s" % (key, path)) - wm.add_watch(path, mask, proc_fun=EventHandler(profile=profile)) + log.debug(u"adding watch to profile '{0}' for folder: {1}".format(key, path)) + watch_manager.add_watch(path, mask, proc_fun=EventHandler(profile=profile)) # Create an action thread for the profile. - name = 'actions-%s' % key - log.debug("FileMonitorDaemon.run: starting action thread: %s" % name) - thread = threading.Thread(target=filemon.perform_actions, - name=name, args=(profile,)) + name = u'actions-{0}'.format(key) + log.debug(u"starting action thread: {0}".format(name)) + thread = Thread(target=perform_actions, name=name, args=(profile,)) thread.daemon = True thread.start() @@ -135,20 +127,16 @@ def get_daemon(config, pidfile=None): """ Get a :class:`FileMonitorDaemon` instance. """ - if pidfile is None: pidfile = config.get('rattail.filemon', 'pid_path', default='/var/run/rattail/filemon.pid') - daemon = FileMonitorDaemon(pidfile) - daemon.config = config - return daemon + return FileMonitorDaemon(pidfile, config=config) def start_daemon(config, pidfile=None, daemonize=True): """ Start the file monitor daemon. """ - get_daemon(config, pidfile).start(daemonize) @@ -156,5 +144,4 @@ def stop_daemon(config, pidfile=None): """ Stop the file monitor daemon. """ - get_daemon(config, pidfile).stop() diff --git a/rattail/filemon/util.py b/rattail/filemon/util.py index e14463ae37612079b0658703802a288f8b4bbb7a..c98e2d166d5a6f4326d76295e9db1ba3d3247ae2 100644 --- a/rattail/filemon/util.py +++ b/rattail/filemon/util.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2012 Lance Edgar +# Copyright © 2010-2014 Lance Edgar # # This file is part of Rattail. # @@ -26,15 +25,40 @@ File Monitor Utilities """ +import os +import logging -def raise_exception(path, message=u"Fake error for testing"): + +log = logging.getLogger(__name__) + + +def queue_existing(profile, path): """ - File monitor action which always raises an exception. + Adds files found in a watched folder to a processing queue. This is called + when the monitor first starts, to handle the case of files which exist + prior to startup. + + If files are found, they are first sorted by modification timestamp, using + a lexical sort on the filename as a tie-breaker, and then added to the + queue in that order. - This is meant to be a simple way to test the error handling of a file - monitor. For example, whether or not file processing continues for - subsequent files after the first error is encountered. If logging - configuration dictates that an email should be sent, it will of course test - that as well. + :param profile: Monitor configuration profile for which the folder is to be + watched. The profile is expected to already have a queue attached; any + existing files will be added to this queue. + + :param path: Folder path which is to be checked for files. """ - raise Exception(u'{0}: {1}'.format(message, path)) + paths = [os.path.join(path, p) for p in os.listdir(path)] + paths = sorted(paths, key=lambda p: (os.path.getmtime(p), p)) + for path in paths: + + # Only process normal files. + if not os.path.isfile(path): + continue + + # If using locks, don't process "in transit" files. + if profile.watch_locks and path.endswith(u'.lock'): + continue + + log.debug(u"queuing existing file: {0}".format(path)) + profile.queue.put(path) diff --git a/rattail/filemon/win32.py b/rattail/filemon/win32.py index 1d5019f2221d9fb955ee83af8915d4708d7b5394..ea7572107d571b913c2b2c837965a6bd2921b3c2 100644 --- a/rattail/filemon/win32.py +++ b/rattail/filemon/win32.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2012 Lance Edgar +# Copyright © 2010-2014 Lance Edgar # # This file is part of Rattail. # @@ -23,37 +22,43 @@ ################################################################################ """ -``rattail.filemon.win32`` -- File Monitor for Windows +File Monitor for Windows """ -import os.path +import os import Queue import logging -from rattail import filemon from rattail.win32.service import Service from rattail.threads import Thread +from rattail.filemon.config import load_profiles +from rattail.filemon.actions import perform_actions +from rattail.filemon.util import queue_existing +# TODO: Would be nice to have a note explaining why this hack exists. name = __name__ -if name == 'win32': - name = 'rattail.filemon.win32' +if name == u'win32': + name = u'rattail.filemon.win32' log = logging.getLogger(name) class RattailFileMonitor(Service): + """ + Windows service implementation of the File Monitor. + """ - _svc_name_ = "RattailFileMonitor" - _svc_display_name_ = "Rattail : File Monitoring Service" - _svc_description_ = ("Monitors one or more folders for incoming files, " - "and performs configured actions as new files arrive.") + _svc_name_ = u"RattailFileMonitor" + _svc_display_name_ = u"Rattail : File Monitoring Service" + _svc_description_ = (u"Monitors one or more folders for incoming files, " + u"and performs configured actions as new files arrive.") def Initialize(self, config): """ Service initialization. """ # Read monitor profile(s) from config. - self.monitored = filemon.get_monitor_profiles(config) + self.monitored = load_profiles(config) # Make sure we have something to do. if not self.monitored: @@ -70,28 +75,28 @@ class RattailFileMonitor(Service): # Maybe put all pre-existing files in the queue. if profile.process_existing: - filemon.queue_existing(profile, path) + queue_existing(profile, path) - # Create a monitor thread for the folder. - name = u'monitor_{0}-{1}'.format(key, i) - log.debug(u"starting '{0}' thread for folder: {1}".format(name, path)) - thread = Thread(target=monitor_files, name=name, args=(profile, path)) + # Create a watcher thread for the folder. + name = u'watcher_{0}-{1}'.format(key, i) + log.debug(u"starting {0} thread for folder: {1}".format(repr(name), repr(path))) + thread = Thread(target=watch_directory, name=name, args=(profile, path)) thread.daemon = True thread.start() # Create an action thread for the profile. name = u'actions_{0}'.format(key) - log.debug(u"starting action thread: {0}".format(name)) - thread = Thread(target=filemon.perform_actions, name=name, args=(profile,)) + log.debug(u"starting action thread: {0}".format(repr(name))) + thread = Thread(target=perform_actions, name=name, args=(profile,)) thread.daemon = True thread.start() return True -def monitor_files(profile, path): +def watch_directory(profile, path): """ - Callable target for file monitor threads. + Callable target for watcher threads. """ import win32file @@ -122,7 +127,7 @@ def monitor_files(profile, path): for action, fname in results: fpath = os.path.join(path, fname) queue = False - if profile.locks: + if profile.watch_locks: if action == winnt.FILE_ACTION_REMOVED and fpath.endswith('.lock'): fpath = fpath[:-5] queue = True diff --git a/tests/filemon/test_actions.py b/tests/filemon/test_actions.py new file mode 100644 index 0000000000000000000000000000000000000000..d53a798d43084839a82ad353e4d3aaef198435a8 --- /dev/null +++ b/tests/filemon/test_actions.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +import os +import time +import Queue +from unittest import TestCase + +from mock import Mock, patch, call +from fixture import TempIO + +from edbob.configuration import AppConfigParser + +from rattail.filemon import actions +from rattail.filemon.config import Profile, ProfileAction + + +class TestAction(TestCase): + + def test_callable_must_be_implemented_in_subclass(self): + config = AppConfigParser(u'rattail') + action = actions.Action(config) + self.assertRaises(NotImplementedError, action) + + +@patch(u'rattail.filemon.actions.noop') +class TestPerformActions(TestCase): + + def setUp(self): + self.tmp = TempIO() + self.config = AppConfigParser(u'rattail') + self.config.set(u'rattail.filemon', u'monitor', u'foo') + self.config.set(u'rattail.filemon', u'foo.dirs', self.tmp) + self.config.set(u'rattail.filemon', u'foo.actions', u'noop') + self.config.set(u'rattail.filemon', u'foo.action.noop.func', u'rattail.filemon.actions:noop') + # Must delay creating the profile since doing it here would bypass our mock of noop. + + def get_profile(self, stop_on_error=False): + profile = Profile(self.config, u'foo') + profile.stop_on_error = stop_on_error + profile.queue = Mock() + profile.queue.get_nowait.side_effect = [ + Queue.Empty, # for coverage sake; will be effectively skipped + self.tmp.putfile(u'file1', u''), + self.tmp.putfile(u'file2', u''), + self.tmp.putfile(u'file3', u''), + actions.StopProcessing, + ] + return profile + + def test_action_is_invoked_for_each_file_in_queue(self, noop): + profile = self.get_profile() + actions.perform_actions(profile) + self.assertEqual(noop.call_count, 3) + noop.assert_has_calls([ + call(self.tmp.join(u'file1')), + call(self.tmp.join(u'file2')), + call(self.tmp.join(u'file3')), + ]) + + def test_action_is_skipped_for_nonexistent_file(self, noop): + profile = self.get_profile() + os.remove(self.tmp.join(u'file2')) + actions.perform_actions(profile) + self.assertEqual(noop.call_count, 2) + # no call for file2 + noop.assert_has_calls([ + call(self.tmp.join(u'file1')), + call(self.tmp.join(u'file3')), + ]) + + def test_action_which_raises_error_causes_subsequent_actions_to_be_skipped_for_same_file(self, noop): + self.config.set(u'rattail.filemon', u'foo.actions', u'noop, delete') + self.config.set(u'rattail.filemon', u'foo.action.delete.func', u'os:remove') + profile = self.get_profile() + # processing second file fails, so it shouldn't be deleted + noop.side_effect = [None, RuntimeError, None] + actions.perform_actions(profile) + self.assertFalse(os.path.exists(self.tmp.join(u'file1'))) + self.assertTrue(os.path.exists(self.tmp.join(u'file2'))) + self.assertFalse(os.path.exists(self.tmp.join(u'file3'))) + + def test_action_which_raises_error_causes_all_processing_to_stop_if_so_configured(self, noop): + self.config.set(u'rattail.filemon', u'foo.actions', u'noop, delete') + self.config.set(u'rattail.filemon', u'foo.action.delete.func', u'os:remove') + profile = self.get_profile(stop_on_error=True) + # processing second file fails; third file shouldn't be processed at all + noop.side_effect = [None, RuntimeError, None] + actions.perform_actions(profile) + self.assertEqual(noop.call_count, 2) + noop.assert_has_calls([ + call(self.tmp.join(u'file1')), + call(self.tmp.join(u'file2')), + ]) + self.assertFalse(os.path.exists(self.tmp.join(u'file1'))) + self.assertTrue(os.path.exists(self.tmp.join(u'file2'))) + self.assertTrue(os.path.exists(self.tmp.join(u'file3'))) + + +class TestInvokeAction(TestCase): + + def setUp(self): + self.action = ProfileAction() + self.action.action = Mock(return_value=None) + self.action.retry_attempts = 6 + self.tmp = TempIO() + self.file = self.tmp.putfile(u'file', u'') + + def test_action_which_succeeds_is_only_called_once(self): + actions.invoke_action(self.action, self.file) + self.assertEqual(self.action.action.call_count, 1) + + def test_action_with_no_delay_does_not_pause_between_attempts(self): + self.action.retry_attempts = 3 + self.action.action.side_effect = [RuntimeError, RuntimeError, None] + start = time.time() + actions.invoke_action(self.action, self.file) + self.assertEqual(self.action.action.call_count, 3) + self.assertTrue(time.time() - start < 1.0) + + def test_action_with_delay_pauses_between_attempts(self): + self.action.retry_attempts = 3 + self.action.retry_delay = 1 + self.action.action.side_effect = [RuntimeError, RuntimeError, None] + start = time.time() + actions.invoke_action(self.action, self.file) + self.assertEqual(self.action.action.call_count, 3) + self.assertTrue(time.time() - start >= 2.0) + + def test_action_which_fails_is_only_attempted_the_specified_number_of_times(self): + self.action.action.side_effect = RuntimeError + # Last attempt will not handle the exception; assert that as well. + self.assertRaises(RuntimeError, actions.invoke_action, self.action, self.file) + self.assertEqual(self.action.action.call_count, 6) + + def test_action_which_fails_then_succeeds_stops_retrying(self): + # First 2 attempts fail, third succeeds. + self.action.action.side_effect = [RuntimeError, RuntimeError, None] + actions.invoke_action(self.action, self.file) + self.assertEqual(self.action.action.call_count, 3) + + def test_action_which_fails_with_different_errors_stops_retrying(self): + self.action.action.side_effect = [ValueError, TypeError, None] + # Second attempt will not handle the exception; assert that as well. + self.assertRaises(TypeError, actions.invoke_action, self.action, self.file) + self.assertEqual(self.action.action.call_count, 2) + + +class TestRaiseException(TestCase): + + def test_exception_is_raised(self): + # this hardly deserves a test, but what the hell + self.assertRaises(Exception, actions.raise_exception, '/dev/null') diff --git a/tests/filemon/test_config.py b/tests/filemon/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..1434fe034314d8e8b6a7fcd3ec536377884f6a27 --- /dev/null +++ b/tests/filemon/test_config.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- + +import os +from unittest import TestCase + +from fixture import TempIO + +from edbob.configuration import AppConfigParser + +from rattail.filemon import config +from rattail.filemon import Action +from rattail.exceptions import ConfigurationError + + +class TestProfile(TestCase): + + def setUp(self): + self.config = AppConfigParser(u'rattail') + self.config.set(u'rattail.filemon', u'foo.actions', u'bar') + + def test_empty_config_means_empty_profile(self): + profile = config.Profile(self.config, u'nonexistent_key') + self.assertEqual(len(profile.dirs), 0) + self.assertFalse(profile.watch_locks) + self.assertTrue(profile.process_existing) + self.assertFalse(profile.stop_on_error) + self.assertEqual(len(profile.actions), 0) + + def test_action_must_specify_callable(self): + self.assertRaises(ConfigurationError, config.Profile, self.config, u'foo') + + def test_action_must_not_specify_both_func_and_class_callables(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.class', u'baz') + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'baz') + self.assertRaises(ConfigurationError, config.Profile, self.config, u'foo') + + def test_action_with_func_callable(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'os:remove') + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + action = profile.actions[0] + self.assertEqual(action.spec, u'os:remove') + self.assertTrue(action.action is os.remove) + + def test_action_with_class_callable(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.class', u'rattail.filemon:Action') + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + action = profile.actions[0] + self.assertEqual(action.spec, u'rattail.filemon:Action') + self.assertTrue(isinstance(action.action, Action)) + + def test_action_with_args(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'shutil:move') + self.config.set(u'rattail.filemon', u'foo.action.bar.args', u'/dev/null') + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + action = profile.actions[0] + self.assertEqual(len(action.args), 1) + self.assertEqual(action.args[0], u'/dev/null') + + def test_action_with_kwargs(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:raise_exception') + self.config.set(u'rattail.filemon', u'foo.action.bar.kwarg.message', u"Hello World") + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + action = profile.actions[0] + self.assertEqual(len(action.kwargs), 1) + self.assertEqual(action.kwargs[u'message'], u"Hello World") + + def test_action_with_default_retry(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:noop') + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + action = profile.actions[0] + self.assertEqual(action.retry_attempts, 1) + self.assertEqual(action.retry_delay, 0) + + def test_action_with_valid_configured_retry(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:noop') + self.config.set(u'rattail.filemon', u'foo.action.bar.retry_attempts', u'42') + self.config.set(u'rattail.filemon', u'foo.action.bar.retry_delay', u'100') + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + action = profile.actions[0] + self.assertEqual(action.retry_attempts, 42) + self.assertEqual(action.retry_delay, 100) + + def test_action_with_invalid_configured_retry(self): + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:noop') + self.config.set(u'rattail.filemon', u'foo.action.bar.retry_attempts', u'-1') + self.config.set(u'rattail.filemon', u'foo.action.bar.retry_delay', u'-1') + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + action = profile.actions[0] + self.assertEqual(action.retry_attempts, 1) + self.assertEqual(action.retry_delay, 0) + + def test_normalize_dirs(self): + tmp = TempIO() + dir1 = tmp.mkdir(u'dir1') + # dir2 will be pruned due to its not existing + dir2 = tmp.mkdir(u'dir2') + os.rmdir(dir2) + # file1 will be pruned due to its not being a directory + file1 = tmp.putfile(u'file1', u'') + self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'os:remove') + self.config.set(u'rattail.filemon', u'foo.dirs', u' '.join([u'"{0}"'.format(d) for d in [dir1, dir2, file1]])) + profile = config.Profile(self.config, u'foo') + self.assertEqual(len(profile.dirs), 1) + self.assertEqual(profile.dirs[0], dir1) + + +class TestLoadProfiles(TestCase): + + def setUp(self): + self.tmp = TempIO() + self.config = AppConfigParser(u'rattail') + self.config.set(u'rattail.filemon', u'monitor', u'foo, bar') + self.config.set(u'rattail.filemon', u'foo.dirs', u'"{0}"'.format(self.tmp)) + self.config.set(u'rattail.filemon', u'foo.actions', u'delete') + self.config.set(u'rattail.filemon', u'foo.action.delete.func', u'os:remove') + self.config.set(u'rattail.filemon', u'bar.dirs', u'"{0}"'.format(self.tmp)) + self.config.set(u'rattail.filemon', u'bar.actions', u'delete') + self.config.set(u'rattail.filemon', u'bar.action.delete.func', u'os:remove') + + def test_returns_all_profiles_specified_in_monitor_option(self): + monitored = config.load_profiles(self.config) + self.assertEqual(len(monitored), 2) + # leave profiles intact but replace monitor option with one key only + self.config.set(u'rattail.filemon', u'monitor', u'foo') + monitored = config.load_profiles(self.config) + self.assertEqual(len(monitored), 1) + + def test_monitor_option_must_be_specified(self): + self.config.remove_option(u'rattail.filemon', u'monitor') + self.assertRaises(ConfigurationError, config.load_profiles, self.config) + + def test_profiles_which_define_no_watched_folders_are_pruned(self): + monitored = config.load_profiles(self.config) + self.assertEqual(len(monitored), 2) + # remove foo's watched folder(s) + self.config.remove_option(u'rattail.filemon', u'foo.dirs') + monitored = config.load_profiles(self.config) + self.assertEqual(len(monitored), 1) + + def test_profiles_which_define_no_actions_are_pruned(self): + monitored = config.load_profiles(self.config) + self.assertEqual(len(monitored), 2) + # remove foo's actions + self.config.remove_option(u'rattail.filemon', u'foo.actions') + monitored = config.load_profiles(self.config) + self.assertEqual(len(monitored), 1) + + def test_fallback_to_legacy_mode(self): + # replace 'monitor' option with 'monitored' and update profiles accordingly + self.config.remove_option(u'rattail.filemon', u'monitor') + self.config.set(u'rattail.filemon', u'monitored', u'foo, bar') + self.config.set(u'rattail.filemon', u'foo.dirs', u"['{0}']".format(self.tmp)) + self.config.set(u'rattail.filemon', u'foo.actions', u"['os:remove']") + self.config.set(u'rattail.filemon', u'bar.dirs', u"['{0}']".format(self.tmp)) + self.config.set(u'rattail.filemon', u'bar.actions', u"['os:remove']") + monitored = config.load_profiles(self.config) + self.assertEqual(len(monitored), 2) + profiles = list(monitored.values()) + self.assertTrue(isinstance(profiles[0], config.LegacyProfile)) + self.assertTrue(isinstance(profiles[1], config.LegacyProfile)) + + +class TestLegacyProfile(TestCase): + + def setUp(self): + self.config = AppConfigParser(u'rattail') + + def test_empty_config_means_empty_profile(self): + profile = config.LegacyProfile(self.config, u'nonexistent_key') + self.assertEqual(len(profile.dirs), 0) + self.assertFalse(profile.watch_locks) + self.assertTrue(profile.process_existing) + self.assertFalse(profile.stop_on_error) + self.assertEqual(len(profile.actions), 0) + + def test_action_with_spec_only(self): + self.config.set(u'rattail.filemon', u'foo.actions', u"['os:remove']") + profile = config.LegacyProfile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + spec, action, args, kwargs = profile.actions[0] + self.assertEqual(spec, u'os:remove') + self.assertTrue(action is os.remove) + + def test_action_with_spec_and_args(self): + self.config.set(u'rattail.filemon', u'foo.actions', u"[('shutil:move', u'/dev/null')]") + profile = config.LegacyProfile(self.config, u'foo') + self.assertEqual(len(profile.actions), 1) + spec, action, args, kwargs = profile.actions[0] + self.assertEqual(spec, u'shutil:move') + self.assertEqual(len(args), 1) + self.assertEqual(args[0], u'/dev/null') + + def test_normalize_dirs(self): + tmp = TempIO() + dir1 = tmp.mkdir(u'dir1') + # dir2 will be pruned due to its not existing + dir2 = tmp.mkdir(u'dir2') + os.rmdir(dir2) + # file1 will be pruned due to its not being a directory + file1 = tmp.putfile(u'file1', u'') + self.config.set(u'rattail.filemon', u'foo.dirs', u"[{0}]".format(u', '.join([u"'{0}'".format(d) for d in [dir1, dir2, file1]]))) + profile = config.LegacyProfile(self.config, u'foo') + self.assertEqual(len(profile.dirs), 1) + self.assertEqual(profile.dirs[0], dir1) + + +class TestLoadLegacyProfiles(TestCase): + + def setUp(self): + self.tmp = TempIO() + self.config = AppConfigParser(u'rattail') + self.config.set(u'rattail.filemon', u'monitored', u'foo, bar') + self.config.set(u'rattail.filemon', u'foo.dirs', u"['{0}']".format(self.tmp)) + self.config.set(u'rattail.filemon', u'foo.actions', u"['os:remove']") + self.config.set(u'rattail.filemon', u'bar.dirs', u"['{0}']".format(self.tmp)) + self.config.set(u'rattail.filemon', u'bar.actions', u"['os:remove']") + + def test_returns_all_profiles_specified_in_monitor_option(self): + monitored = config.load_legacy_profiles(self.config) + self.assertEqual(len(monitored), 2) + # leave profiles intact but replace monitored option with one key only + self.config.set(u'rattail.filemon', u'monitored', u'foo') + monitored = config.load_legacy_profiles(self.config) + self.assertEqual(len(monitored), 1) + + def test_monitor_option_must_be_specified(self): + self.config.remove_option(u'rattail.filemon', u'monitored') + self.assertRaises(ConfigurationError, config.load_legacy_profiles, self.config) + + def test_profiles_which_define_no_watched_folders_are_pruned(self): + monitored = config.load_legacy_profiles(self.config) + self.assertEqual(len(monitored), 2) + # remove foo's watched folder(s) + self.config.remove_option(u'rattail.filemon', u'foo.dirs') + monitored = config.load_legacy_profiles(self.config) + self.assertEqual(len(monitored), 1) + + def test_profiles_which_define_no_actions_are_pruned(self): + monitored = config.load_legacy_profiles(self.config) + self.assertEqual(len(monitored), 2) + # remove foo's actions + self.config.remove_option(u'rattail.filemon', u'foo.actions') + monitored = config.load_legacy_profiles(self.config) + self.assertEqual(len(monitored), 1) + + +class TestParseList(TestCase): + + def test_none(self): + value = config.parse_list(None) + self.assertEqual(len(value), 0) + + def test_single_value(self): + value = config.parse_list(u'foo') + self.assertEqual(len(value), 1) + self.assertEqual(value[0], u'foo') + + def test_single_value_padded_by_spaces(self): + value = config.parse_list(u' foo ') + self.assertEqual(len(value), 1) + self.assertEqual(value[0], u'foo') + + def test_slash_is_not_a_separator(self): + value = config.parse_list(u'/dev/null') + self.assertEqual(len(value), 1) + self.assertEqual(value[0], u'/dev/null') + + def test_multiple_values_separated_by_whitespace(self): + value = config.parse_list(u'foo bar baz') + self.assertEqual(len(value), 3) + self.assertEqual(value[0], u'foo') + self.assertEqual(value[1], u'bar') + self.assertEqual(value[2], u'baz') + + def test_multiple_values_separated_by_commas(self): + value = config.parse_list(u'foo,bar,baz') + self.assertEqual(len(value), 3) + self.assertEqual(value[0], u'foo') + self.assertEqual(value[1], u'bar') + self.assertEqual(value[2], u'baz') + + def test_multiple_values_separated_by_whitespace_and_commas(self): + value = config.parse_list(u' foo, bar baz') + self.assertEqual(len(value), 3) + self.assertEqual(value[0], u'foo') + self.assertEqual(value[1], u'bar') + self.assertEqual(value[2], u'baz') + + def test_multiple_values_separated_by_whitespace_and_commas_with_some_quoting(self): + value = config.parse_list(u""" + foo + "C:\\some path\\with spaces\\and, a comma", + baz +""") + self.assertEqual(len(value), 3) + self.assertEqual(value[0], u'foo') + self.assertEqual(value[1], u'C:\\some path\\with spaces\\and, a comma') + self.assertEqual(value[2], u'baz') diff --git a/tests/filemon/test_linux.py b/tests/filemon/test_linux.py new file mode 100644 index 0000000000000000000000000000000000000000..7bcf31a162f98cb6c8bba8e95d56d8d50f8056eb --- /dev/null +++ b/tests/filemon/test_linux.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +import Queue +from unittest import TestCase + +from mock import Mock +from fixture import TempIO + +from edbob.configuration import AppConfigParser + +from rattail.filemon import linux +from rattail.filemon.config import Profile + + +class TestEventHandler(TestCase): + + def setUp(self): + self.tmp = TempIO() + self.config = AppConfigParser(u'rattail') + self.config.set(u'rattail.filemon', u'monitor', u'foo') + self.config.set(u'rattail.filemon', u'foo.dirs', self.tmp) + self.config.set(u'rattail.filemon', u'foo.actions', u'noop') + self.config.set(u'rattail.filemon', u'foo.action.noop.func', u'rattail.filemon.actions:noop') + self.profile = Profile(self.config, u'foo') + self.profile.queue = Queue.Queue() + self.handler = linux.EventHandler() + self.handler.my_init(self.profile) + + def test_in_access_event_does_nothing(self): + event = Mock(pathname=self.tmp.putfile(u'file', u'')) + self.handler.process_IN_ACCESS(event) + self.assertTrue(self.profile.queue.empty()) + + def test_in_attrib_event_does_nothing(self): + event = Mock(pathname=self.tmp.putfile(u'file', u'')) + self.handler.process_IN_ATTRIB(event) + self.assertTrue(self.profile.queue.empty()) + + def test_in_create_event_does_nothing(self): + event = Mock(pathname=self.tmp.putfile(u'file', u'')) + self.handler.process_IN_CREATE(event) + self.assertTrue(self.profile.queue.empty()) + + def test_in_modify_event_does_nothing(self): + event = Mock(pathname=self.tmp.putfile(u'file', u'')) + self.handler.process_IN_MODIFY(event) + self.assertTrue(self.profile.queue.empty()) + + def test_in_close_write_event_queues_file_if_profile_does_not_watch_locks(self): + event = Mock(pathname=self.tmp.putfile(u'file', u'')) + self.profile.watch_locks = False + self.handler.process_IN_CLOSE_WRITE(event) + self.assertEqual(self.profile.queue.qsize(), 1) + self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file')) + + def test_in_close_write_event_does_nothing_if_profile_watches_locks(self): + event = Mock(pathname=self.tmp.putfile(u'file.lock', u'')) + self.profile.watch_locks = True + self.handler.process_IN_CLOSE_WRITE(event) + self.assertTrue(self.profile.queue.empty()) + + def test_in_moved_to_event_queues_file_if_profile_does_not_watch_locks(self): + event = Mock(pathname=self.tmp.putfile(u'file', u'')) + self.profile.watch_locks = False + self.handler.process_IN_MOVED_TO(event) + self.assertEqual(self.profile.queue.qsize(), 1) + self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file')) + + def test_in_moved_to_event_does_nothing_if_profile_watches_locks(self): + event = Mock(pathname=self.tmp.putfile(u'file.lock', u'')) + self.profile.watch_locks = True + self.handler.process_IN_MOVED_TO(event) + self.assertTrue(self.profile.queue.empty()) + + def test_in_delete_event_queues_file_if_profile_watches_locks(self): + event = Mock(pathname=self.tmp.putfile(u'file.lock', u'')) + self.profile.watch_locks = True + self.handler.process_IN_DELETE(event) + self.assertEqual(self.profile.queue.qsize(), 1) + self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file')) + + def test_in_moved_to_event_does_nothing_if_profile_does_not_watch_locks(self): + event = Mock(pathname=self.tmp.putfile(u'file', u'')) + self.profile.watch_locks = False + self.handler.process_IN_DELETE(event) + self.assertTrue(self.profile.queue.empty()) diff --git a/tests/filemon/test_util.py b/tests/filemon/test_util.py index f9ed406539265459e2e0232a6e0a6894efe0357a..c0b9e211c3f0c12998b48663686403b247ac2f6d 100644 --- a/tests/filemon/test_util.py +++ b/tests/filemon/test_util.py @@ -1,15 +1,45 @@ +# -*- coding: utf-8 -*- +import Queue from unittest import TestCase +from fixture import TempIO + +from edbob.configuration import AppConfigParser + from rattail.filemon import util +from rattail.filemon.config import Profile + + +class TestQueueExisting(TestCase): + + def setUp(self): + self.tmp = TempIO() + self.config = AppConfigParser(u'rattail') + self.config.set(u'rattail.filemon', u'monitor', u'foo') + self.config.set(u'rattail.filemon', u'foo.dirs', self.tmp) + self.config.set(u'rattail.filemon', u'foo.actions', u'noop') + self.config.set(u'rattail.filemon', u'foo.action.noop.func', u'rattail.filemon.actions:noop') + self.profile = Profile(self.config, u'foo') + self.profile.queue = Queue.Queue() + def test_nothing_queued_if_no_files_exist(self): + util.queue_existing(self.profile, self.tmp) + self.assertTrue(self.profile.queue.empty()) -class TestRaiseException(TestCase): + def test_normal_files_are_queued_but_not_folders(self): + self.tmp.putfile(u'file', u'') + self.tmp.mkdir(u'folder') + util.queue_existing(self.profile, self.tmp) + self.assertEqual(self.profile.queue.qsize(), 1) + self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file')) + self.assertTrue(self.profile.queue.empty()) - def test_exception_is_raised_and_has_correct_message(self): - path = u'fake_path' - self.assertRaises(Exception, util.raise_exception, path) - try: - util.raise_exception(path, message=u"This is my custom message") - except Exception as error: - self.assertEqual(unicode(error), u"This is my custom message: fake_path") + def test_if_profile_watches_locks_then_normal_files_are_queued_but_not_lock_files(self): + self.profile.watch_locks = True + self.tmp.putfile(u'file1.lock', u'') + self.tmp.putfile(u'file2', u'') + util.queue_existing(self.profile, self.tmp) + self.assertEqual(self.profile.queue.qsize(), 1) + self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file2')) + self.assertTrue(self.profile.queue.empty())