diff --git a/.gitignore b/.gitignore
index e87630d909a6d3401a5943ecc83c0578bf416b49..c0c5008cfe6a15bcbae010550d098e2d14a3e07f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
.tox/
+docs/_build/
rattail.egg-info/
diff --git a/docs/api/rattail/filemon/actions.rst b/docs/api/rattail/filemon/actions.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d5e248864783ce5c089394eecab42c31e93e6cd0
--- /dev/null
+++ b/docs/api/rattail/filemon/actions.rst
@@ -0,0 +1,7 @@
+.. -*- coding: utf-8 -*-
+
+``rattail.filemon.actions``
+===========================
+
+.. automodule:: rattail.filemon.actions
+ :members:
diff --git a/docs/api/rattail/filemon/config.rst b/docs/api/rattail/filemon/config.rst
new file mode 100644
index 0000000000000000000000000000000000000000..993da0103c3dfff1b0ba371b08464c3f1f281b85
--- /dev/null
+++ b/docs/api/rattail/filemon/config.rst
@@ -0,0 +1,7 @@
+.. -*- coding: utf-8 -*-
+
+``rattail.filemon.config``
+==========================
+
+.. automodule:: rattail.filemon.config
+ :members:
diff --git a/docs/api/rattail/filemon/index.rst b/docs/api/rattail/filemon/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..0ccaa85e0c25150ca39fc24f882f243a1d48b310
--- /dev/null
+++ b/docs/api/rattail/filemon/index.rst
@@ -0,0 +1,8 @@
+.. -*- coding: utf-8 -*-
+
+``rattail.filemon``
+===================
+
+.. automodule:: rattail.filemon
+
+.. autoclass:: Action
diff --git a/docs/api/rattail/filemon/linux.rst b/docs/api/rattail/filemon/linux.rst
new file mode 100644
index 0000000000000000000000000000000000000000..832458a53eb5035fd5ab6b58f29cf5930b11a541
--- /dev/null
+++ b/docs/api/rattail/filemon/linux.rst
@@ -0,0 +1,7 @@
+.. -*- coding: utf-8 -*-
+
+``rattail.filemon.linux``
+=========================
+
+.. automodule:: rattail.filemon.linux
+ :members:
diff --git a/docs/api/rattail/filemon/util.rst b/docs/api/rattail/filemon/util.rst
index 64ae9206b221591c640b07905d51ea3e4c19eaf2..9c04520eac57bf07abf5b2d940d096dd69ddaca5 100644
--- a/docs/api/rattail/filemon/util.rst
+++ b/docs/api/rattail/filemon/util.rst
@@ -1,3 +1,4 @@
+.. -*- coding: utf-8 -*-
``rattail.filemon.util``
========================
diff --git a/docs/api/rattail/filemon/win32.rst b/docs/api/rattail/filemon/win32.rst
new file mode 100644
index 0000000000000000000000000000000000000000..9ed13618110a8b30b3f765b9d117ea5d86016570
--- /dev/null
+++ b/docs/api/rattail/filemon/win32.rst
@@ -0,0 +1,7 @@
+.. -*- coding: utf-8 -*-
+
+``rattail.filemon.win32``
+=========================
+
+.. automodule:: rattail.filemon.win32
+ :members:
diff --git a/docs/index.rst b/docs/index.rst
index c60c6004603a822f8a70c7daae118e490637a0f5..83f49523667724b27a20f2a19678e71af5bfd4a3 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -20,7 +20,12 @@ Package API:
api/rattail/db/util
api/rattail/enum
api/rattail/exceptions
+ api/rattail/filemon/index
+ api/rattail/filemon/actions
+ api/rattail/filemon/config
+ api/rattail/filemon/linux
api/rattail/filemon/util
+ api/rattail/filemon/win32
api/rattail/logging
diff --git a/rattail/exceptions.py b/rattail/exceptions.py
index d6b1dd559cb1f0e5b17485b62dc61bc99534a8ea..e27dfa3880e66d1f8462d733525cff326ac4a6a4 100644
--- a/rattail/exceptions.py
+++ b/rattail/exceptions.py
@@ -1,9 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
################################################################################
#
# Rattail -- Retail Software Framework
-# Copyright © 2010-2012 Lance Edgar
+# Copyright © 2010-2014 Lance Edgar
#
# This file is part of Rattail.
#
@@ -35,6 +34,12 @@ class RattailError(Exception):
"""
+class ConfigurationError(RattailError):
+ """
+ Generic class for configuration errors.
+ """
+
+
class BatchError(RattailError):
"""
Base class for all batch-related errors.
diff --git a/rattail/filemon/__init__.py b/rattail/filemon/__init__.py
index c4456bfb90aaa1cb2ba4a0bdffe3443d1f28e04a..58c11c9b76121ceddfcd494e4ac061477e1259cf 100644
--- a/rattail/filemon/__init__.py
+++ b/rattail/filemon/__init__.py
@@ -1,9 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
################################################################################
#
# Rattail -- Retail Software Framework
-# Copyright © 2010-2012 Lance Edgar
+# Copyright © 2010-2014 Lance Edgar
#
# This file is part of Rattail.
#
@@ -23,204 +22,7 @@
################################################################################
"""
-``rattail.filemon`` -- File Monitor
+File Monitor
"""
-import os
-import os.path
-import sys
-import Queue
-import logging
-
-from ..util import load_object
-
-if sys.platform == 'win32':
- import win32api
- from rattail.win32 import file_is_free
-
-
-log = logging.getLogger(__name__)
-
-
-class MonitorProfile(object):
- """
- This is a simple profile class, used to represent configuration of the file
- monitor service.
- """
-
- def __init__(self, config, key):
- self.config = config
- self.key = key
-
- self.dirs = config.require('rattail.filemon', '{0}.dirs'.format(key))
- self.dirs = eval(self.dirs)
-
- actions = config.require('rattail.filemon', '{0}.actions'.format(key))
- actions = eval(actions)
-
- self.actions = []
- for action in actions:
- if isinstance(action, tuple):
- spec = action[0]
- args = list(action[1:])
- else:
- spec = action
- args = []
- func = load_object(spec)
- self.actions.append((spec, func, args))
-
- self.locks = config.getboolean(
- 'rattail.filemon', '{0}.locks'.format(key), default=False)
-
- self.process_existing = config.getboolean(
- 'rattail.filemon', '{0}.process_existing'.format(key), default=True)
-
- self.stop_on_error = config.getboolean(
- 'rattail.filemon', '{0}.stop_on_error'.format(key), default=False)
-
-
-def get_monitor_profiles(config):
- """
- Convenience function to load monitor profiles from config.
- """
-
- monitored = {}
-
- # Read monitor profile(s) from config.
- keys = config.require('rattail.filemon', 'monitored')
- keys = keys.split(',')
- for key in keys:
- key = key.strip()
- log.debug("get_monitor_profiles: loading profile: {0}".format(key))
- profile = MonitorProfile(config, key)
- monitored[key] = profile
- for path in profile.dirs[:]:
-
- # Ensure the monitored path exists.
- if not os.path.exists(path):
- log.warning("get_monitor_profiles: profile '{0}' has nonexistent "
- "path, which will be pruned: {1}".format(key, path))
- profile.dirs.remove(path)
-
- # Ensure the monitored path is a folder.
- elif not os.path.isdir(path):
- log.warning("get_monitor_profiles: profile '{0}' has non-folder "
- "path, which will be pruned: {1}".format(key, path))
- profile.dirs.remove(path)
-
- for key in monitored.keys():
- profile = monitored[key]
-
- # Prune any profiles with no valid folders to monitor.
- if not profile.dirs:
- log.warning("get_monitor_profiles: profile has no folders to "
- "monitor, and will be pruned: {0}".format(key))
- del monitored[key]
-
- # Prune any profiles with no valid actions to perform.
- elif not profile.actions:
- log.warning("get_monitor_profiles: profile has no actions to "
- "perform, and will be pruned: {0}".format(key))
- del monitored[key]
-
- return monitored
-
-
-def queue_existing(profile, path):
- """
- Adds files found in a watched folder to a processing queue. This is called
- when the monitor first starts, to handle the case of files which exist
- prior to startup.
-
- If files are found, they are first sorted by modification timestamp, using
- a lexical sort on the filename as a tie-breaker, and then added to the
- queue in that order.
-
- :param profile: Monitor profile for which the folder is to be watched. The
- profile is expected to already have a queue attached; any existing files
- will be added to this queue.
- :type profile: :class:`rattail.filemon.MonitorProfile` instance
-
- :param path: Folder path which is to be checked for files.
- :type path: string
-
- :returns: ``None``
- """
-
- def sorter(x, y):
- mtime_x = os.path.getmtime(x)
- mtime_y = os.path.getmtime(y)
- if mtime_x < mtime_y:
- return -1
- if mtime_x > mtime_y:
- return 1
- return cmp(x, y)
-
- paths = [os.path.join(path, x) for x in os.listdir(path)]
- for path in sorted(paths, cmp=sorter):
-
- # Only process normal files.
- if not os.path.isfile(path):
- continue
-
- # If using locks, don't process "in transit" files.
- if profile.locks and path.endswith('.lock'):
- continue
-
- log.debug("queue_existing: queuing existing file for "
- "profile '{0}': {1}".format(profile.key, path))
- profile.queue.put(path)
-
-
-def perform_actions(profile):
- """
- Callable target for action threads.
- """
-
- keep_going = True
- while keep_going:
-
- try:
- path = profile.queue.get_nowait()
- except Queue.Empty:
- pass
- else:
-
- # In some cases, processing one file may cause other related files
- # to also be processed. When this happens, a path on the queue may
- # point to a file which no longer exists.
- if not os.path.exists(path):
- log.info("perform_actions: path does not exist: {0}".format(path))
- continue
-
- log.debug("perform_actions: processing file: {0}".format(path))
-
- if sys.platform == 'win32':
- while not file_is_free(path):
- win32api.Sleep(0)
-
- for spec, func, args in profile.actions:
-
- log.info("perform_actions: calling function '{0}' on file: {1}".format(
- spec, path))
-
- try:
- func(path, *args)
-
- except:
- log.exception(u"error processing file: %s", path)
-
- # Don't process any more files if the profile is so
- # configured.
- if profile.stop_on_error:
- keep_going = False
-
- # Either way this particular file probably shouldn't be
- # processed any further.
- log.warning("perform_actions: no further processing will "
- "be done for file: {0}".format(path))
- break
-
- log.warning("perform_actions: error encountered, and configuration "
- "dictates that no more actions will be processed for "
- "profile: {0}".format(profile.key))
+from rattail.filemon.actions import Action
diff --git a/rattail/filemon/actions.py b/rattail/filemon/actions.py
new file mode 100644
index 0000000000000000000000000000000000000000..a7db188f61bbdc8f2b5f76a2a4b9b5b365ac46e1
--- /dev/null
+++ b/rattail/filemon/actions.py
@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+################################################################################
+#
+# Rattail -- Retail Software Framework
+# Copyright © 2010-2014 Lance Edgar
+#
+# This file is part of Rattail.
+#
+# Rattail is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Rattail. If not, see .
+#
+################################################################################
+
+"""
+File Monitor Actions
+"""
+
+import os
+import sys
+import time
+import Queue
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+class Action(object):
+ """
+ Base class for file monitor actions.
+ """
+
+ def __init__(self, config):
+ self.config = config
+
+ def __call__(self, *args, **kwargs):
+ """
+ This method must be implemented in the subclass; it defines what the
+ action actually *does*. The file monitor will invoke this method for
+ all new files which are discovered.
+ """
+ raise NotImplementedError
+
+
+class StopProcessing(Exception):
+ """
+ Simple exception to indicate action processing should stop. This is really
+ only useful for tests.
+ """
+
+
+def perform_actions(profile):
+ """
+ Target for action threads. Provides the main loop which checks the queue
+ for new files and invokes actions for each, as they appear.
+ """
+
+ # If running on Windows, we add a step to help ensure the file is truly
+ # free of competing process interests. (In fact it would be nice to have
+ # this on Linux as well, but I'm not sure how to do it there.)
+ wait_for_file = lambda p: p
+ if sys.platform.startswith(u'win'): # pragma: no cover
+ import win32api
+ from rattail.win32 import file_is_free
+ def wait_for_file(path):
+ while not file_is_free(path):
+ win32api.Sleep(0)
+
+ stop = False
+ while not stop:
+
+ try:
+ path = profile.queue.get_nowait()
+ except Queue.Empty:
+ pass
+ except StopProcessing:
+ stop = True
+ else:
+ log.debug(u"queue contained a file: {0}".format(repr(path)))
+
+ # In some cases, processing one file may cause other related files
+ # to also be processed. When this happens, a path on the queue may
+ # point to a file which no longer exists.
+ if not os.path.exists(path):
+ log.warning(u"file path does not exist: {0}".format(path))
+ continue
+
+ # This does nothing unless running on Windows.
+ wait_for_file(path)
+
+ for action in profile.actions:
+ try:
+ invoke_action(action, path)
+
+ except:
+ # Stop processing files altogether for this profile if it
+ # is so configured.
+ if profile.stop_on_error:
+ log.warning(u"an error was encountered, and configuration dictates that no more "
+ u"actions will be processed for profile {0}".format(repr(profile.key)))
+ stop = True
+
+ # Either way no more actions should be invoked for this
+ # particular file.
+ break
+
+
+def invoke_action(action, path):
+ """
+ Invoke a single action on a file, retrying as necessary.
+ """
+ attempts = 0
+ errtype = None
+ while True:
+ attempts += 1
+ log.debug(u"invoking action {0} (attempt #{1} of {2}) on file: {3}".format(
+ repr(action.spec), attempts, action.retry_attempts, repr(path)))
+
+ try:
+ action.action(path, *action.args, **action.kwargs)
+
+ except Exception as error:
+
+ # If we've reached our final attempt, stop retrying.
+ if attempts >= action.retry_attempts:
+ log.exception(u"attempt #{0} failed for action {1} (giving up) on "
+ u"file: {2}".format(attempts, repr(action.spec), repr(path)))
+ raise
+
+ # If this exception is not the first, and is of a different type
+ # than seen previously, do *not* continue to retry.
+ if errtype is not None and not isinstance(error, errtype):
+ log.exception(u"new exception differs from previous one(s), giving up on "
+ u"action {0} for file: {1}".format(repr(action.spec), repr(path)))
+ raise
+
+ # Record the type of exception seen, and pause for next retry.
+ log.warning(u"attempt #{0} failed for action {1} on file: {2}".format(
+ attempts, repr(action.spec), repr(path)), exc_info=True)
+ errtype = type(error)
+ log.debug(u"pausing for {0} seconds before making attempt #{1} of {2}".format(
+ action.retry_delay, attempts + 1, action.retry_attempts))
+ if action.retry_delay:
+ time.sleep(action.retry_delay)
+
+ else:
+ # No error, invocation successful.
+ log.debug(u"attempt #{0} succeeded for action {1} on file: {2}".format(
+ attempts, repr(action.spec), repr(path)))
+ break
+
+
+def raise_exception(path, message=u"Fake error for testing"):
+ """
+ File monitor action which always raises an exception.
+
+ This is meant to be a simple way to test the error handling of a file
+ monitor. For example, whether or not file processing continues for
+ subsequent files after the first error is encountered. If logging
+ configuration dictates that an email should be sent, it will of course test
+ that as well.
+ """
+ raise Exception(u'{0}: {1}'.format(message, path))
+
+
+def noop(path):
+ """
+ File monitor action which does nothing at all.
+
+ This exists for the sake of tests. I doubt it's useful in any other
+ context.
+ """
diff --git a/rattail/filemon/config.py b/rattail/filemon/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..0cee9888e588dfc6e70784d109feb24b17e0f577
--- /dev/null
+++ b/rattail/filemon/config.py
@@ -0,0 +1,379 @@
+# -*- coding: utf-8 -*-
+################################################################################
+#
+# Rattail -- Retail Software Framework
+# Copyright © 2010-2014 Lance Edgar
+#
+# This file is part of Rattail.
+#
+# Rattail is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Rattail. If not, see .
+#
+################################################################################
+
+"""
+File Monitor Configuration
+"""
+
+import os
+import re
+import shlex
+import warnings
+import logging
+
+from rattail.util import load_object
+from rattail.exceptions import ConfigurationError
+
+
+log = logging.getLogger(__name__)
+
+
+class ProfileAction(object):
+ """
+ Simple class to hold configuration for a particular action defined within a
+ monitor :class:`Profile`. Each instance has the following attributes:
+
+ .. attribute:: spec
+
+ The original "spec" string used to obtain the action callable.
+
+ .. attribute:: action
+
+ A reference to the action callable.
+
+ .. attribute:: args
+
+ A sequence of positional arguments to be passed to the callable (in
+ addition to the file path) when invoking the action.
+
+ .. attribute:: kwargs
+
+ A dictionary of keyword arguments to be passed to the callable (in
+ addition to the positional arguments) when invoking the action.
+
+ .. attribute:: retry_attempts
+
+ Number of attempts to make when invoking the action. Defaults to ``1``,
+ meaning the first attempt will be made but no retries will happen.
+
+ .. attribute:: retry_delay
+
+ Number of seconds to pause between retry attempts, if
+ :attr:`retry_attempts` is greater than one. Defaults to ``0``.
+ """
+
+ spec = None
+ action = None
+ args = []
+ kwargs = {}
+ retry_attempts = 1
+ retry_delay = 0
+
+
+class Profile(object):
+ """
+ Simple class to hold configuration for a file monitor "profile". The
+ profile determines which folders to watch, which actions to take when new
+ files appear, etc. Each instance of this class has the following
+ attributes:
+
+ .. attribute:: config
+
+ Reference to the underlying configuration object from which the profile
+ derives its other attributes.
+
+ .. attribute:: key
+
+ String which differentiates this profile from any others which may exist
+ within the configuration.
+
+ .. attribute:: dirs
+
+ List of directory paths which should be watched by the file monitor on
+ behalf of this profile.
+
+ .. attribute:: watch_locks
+
+ Whether the file monitor should watch for new files, or disappearance of
+ file locks.
+
+ If ``False`` (the default), the file monitor will not use any special
+ logic and will simply be on the lookout for any new files to appear;
+ when they do, the configured action(s) will be invoked.
+
+ If this setting is ``True``, then the simple appearance of a new file
+ will not in itself cause any action(s) being taken. Instead, the file
+ monitor will only watch for the *disappearance* of file "locks",
+ i.e. any file or directory whose name ends in ``".lock"``. When the
+ disappearance of such a lock is noticed, the configured action(s) will
+ be invoked.
+
+ .. attribute:: process_existing
+
+ Whether any pre-existing files present in the watched folder(s) should
+ be automatically added to the processing queue.
+
+ If this is ``True`` (the default), then when the file monitor first
+ starts, it will examine each watched folder to see if it already
+ contains files. If it does, then each file will be added to the
+ processing queue exactly as if the monitor had just noticed the file
+ appear. The files will be added in order of creation timestam, in an
+ effort to be consistent with the real-time behavior.
+
+ If this setting is ``False``, any files which happen to exist when the
+ file monitor first starts will be ignored.
+
+ .. attribute:: stop_on_error
+
+ Whether the file monitor should halt processing for all future files, if
+ a single error is encountered.
+
+ If this is ``True``, then if a single file action raises an exception,
+ all subsequent files to appear will be effectively ignored. No further
+ actions will be taken for any of them.
+
+ If this is ``False`` (the default), then a single file error will only
+ cause processing to halt for that particular file. Subsequent files
+ which appear will still be processed as normal.
+
+ .. attribute:: actions
+
+ List of :class:`ProfileAction` instances representing the actions to be
+ invoked when new files are discovered.
+ """
+
+ def __init__(self, config, key):
+ self.config = config
+ self.key = key
+
+ self.dirs = self.normalize_dirs(self._config_list(u'dirs'))
+ self.watch_locks = self._config_boolean(u'watch_locks', False)
+ self.process_existing = self._config_boolean(u'process_existing', True)
+ self.stop_on_error = self._config_boolean(u'stop_on_error', False)
+
+ self.actions = []
+ for action in self._config_list(u'actions'):
+ self.actions.append(self._config_action(action))
+
+ def _config_action(self, name):
+ function = self._config_string(u'action.{0}.func'.format(name))
+ class_ = self._config_string(u'action.{0}.class'.format(name))
+
+ if function and class_:
+ raise ConfigurationError(
+ u"File monitor profile '{0}' has both function *and* class defined for "
+ u"action '{1}' (must have one or the other).".format(self.key, name))
+
+ if not function and not class_:
+ raise ConfigurationError(
+ u"File monitor profile '{0}' has neither function *nor* class defined for "
+ u"action '{1}' (must have one or the other).".format(self.key, name))
+
+ action = ProfileAction()
+
+ if function:
+ action.spec = function
+ action.action = load_object(action.spec)
+ else:
+ action.spec = class_
+ action.action = load_object(action.spec)(self.config)
+
+ action.args = self._config_list(u'action.{0}.args'.format(name))
+
+ action.kwargs = {}
+ pattern = re.compile(ur'^{0}\.action\.{1}\.kwarg\.(?P\w+)$'.format(self.key, name), re.IGNORECASE)
+ for option in self.config.options(u'rattail.filemon'):
+ match = pattern.match(option)
+ if match:
+ action.kwargs[match.group(u'keyword')] = self.config.get(u'rattail.filemon', option)
+
+ action.retry_attempts = self._config_int(u'action.{0}.retry_attempts'.format(name), minimum=1)
+ action.retry_delay = self._config_int(u'action.{0}.retry_delay'.format(name), minimum=0)
+ return action
+
+ def _config_boolean(self, option, default=None):
+ return self.config.getboolean(
+ u'rattail.filemon', u'{0}.{1}'.format(self.key, option), default=default)
+
+ def _config_int(self, option, minimum=0):
+ option = u'{0}.{1}'.format(self.key, option)
+ if self.config.has_option(u'rattail.filemon', option):
+ value = self.config.getint(u'rattail.filemon', option)
+ if value < minimum:
+ log.warning(u"config value {0} is too small; falling back to minimum "
+ u"of {1} for option: {2}".format(value, minimum, option))
+ value = minimum
+ else:
+ value = minimum
+ return value
+
+ def _config_list(self, option):
+ return parse_list(self._config_string(option))
+
+ def _config_string(self, option):
+ return self.config.get(u'rattail.filemon', u'{0}.{1}'.format(self.key, option))
+
+ def normalize_dirs(self, dirs):
+ """
+ Normalize a list of directory paths. Converts all to absolute paths,
+ and prunes those which do not exist or do not refer to directories.
+ """
+ normalized = []
+ for path in dirs:
+ path = os.path.abspath(path)
+ if not os.path.exists(path):
+ log.warning(u"pruning folder which does not exist for profile "
+ u"{0}: {1}".format(repr(self.key), repr(path)))
+ continue
+ if not os.path.isdir(path):
+ log.warning(u"pruning folder which is not a directory for profile "
+ u"{0}: {1}".format(repr(self.key), repr(path)))
+ continue
+ normalized.append(path)
+ return normalized
+
+
+def load_profiles(config):
+ """
+ Load all active file monitor profiles defined within configuration.
+ """
+ # Make sure we have a top-level directive.
+ keys = config.get(u'rattail.filemon', u'monitor')
+ if not keys:
+
+ # No? What about with the old syntax?
+ keys = config.get(u'rattail.filemon', u'monitored')
+ if keys:
+ return load_legacy_profiles(config)
+
+ # Still no? That's no good..
+ raise ConfigurationError(
+ u"The file monitor configuration does not specify any profiles "
+ u"to be monitored. Please defined the 'monitor' option within "
+ u"the [rattail.filemon] section of your config file.")
+
+ monitored = {}
+ for key in parse_list(keys):
+ profile = Profile(config, key)
+ if not profile.dirs:
+ log.warning(u"profile '{0}' has no valid directories to watch".format(key))
+ continue
+ if not profile.actions:
+ log.warning(u"profile '{0}' has no valid actions to invoke".format(key))
+ continue
+ monitored[key] = profile
+
+ return monitored
+
+
+class LegacyProfile(Profile):
+ """
+ This is a file monitor profile corresponding to the first generation of
+ configuration syntax. E.g. it only supports function-based actions, and
+ does not support keyword arguments.
+
+ .. note::
+ This profile type will be deprecated at some point in the future. All
+ new code should assume use :class:`Profile` instead.
+ """
+
+ def __init__(self, config, key):
+ self.config = config
+ self.key = key
+
+ dirs = self._config_string(u'dirs')
+ if dirs:
+ self.dirs = eval(dirs)
+ else:
+ self.dirs = []
+ self.dirs = self.normalize_dirs(self.dirs)
+
+ actions = self._config_string(u'actions')
+ if actions:
+ actions = eval(actions)
+ else:
+ actions = []
+
+ self.actions = []
+ for action in actions:
+ if isinstance(action, tuple):
+ spec = action[0]
+ args = list(action[1:])
+ else:
+ spec = action
+ args = []
+ action = load_object(spec)
+ self.actions.append((spec, action, args, {}))
+
+ self.watch_locks = self._config_boolean(u'locks', False)
+ self.process_existing = self._config_boolean(u'process_existing', True)
+ self.stop_on_error = self._config_boolean(u'stop_on_error', False)
+
+
+def load_legacy_profiles(config):
+ """
+ Load all active *legacy* file monitor profiles defined within
+ configuration.
+ """
+ warnings.warn(u"The use of legacy profiles is deprecated, and should be "
+ u"avoided. Please update your configuration to use the new "
+ u"syntax as soon as possible.", DeprecationWarning)
+
+ monitored = {}
+
+ # Read monitor profile(s) from config.
+ keys = config.get(u'rattail.filemon', u'monitored')
+ if not keys:
+ raise ConfigurationError(
+ u"The file monitor configuration does not specify any profiles "
+ u"to be monitored. Please defined the 'monitor' option within "
+ u"the [rattail.filemon] section of your config file.")
+ keys = keys.split(u',')
+ for key in keys:
+ key = key.strip()
+ log.debug(u"loading profile: {0}".format(key))
+ monitored[key] = LegacyProfile(config, key)
+
+ for key in monitored.keys():
+ profile = monitored[key]
+
+ # Prune any profiles with no valid folders to monitor.
+ if not profile.dirs:
+ log.warning(u"profile has no folders to monitor, "
+ u"and will be pruned: {0}".format(key))
+ del monitored[key]
+
+ # Prune any profiles with no valid actions to perform.
+ elif not profile.actions:
+ log.warning(u"profile has no actions to perform, "
+ u"and will be pruned: {0}".format(key))
+ del monitored[key]
+
+ return monitored
+
+
+def parse_list(value):
+ """
+ Parse a configuration value, splitting by whitespace and/or commas and
+ taking quoting into account, etc., yielding a list of strings.
+ """
+ if value is None:
+ return []
+ parser = shlex.shlex(value)
+ parser.whitespace += u','
+ parser.whitespace_split = True
+ values = list(parser)
+ for i, value in enumerate(values):
+ if value.startswith(u'"') and value.endswith(u'"'):
+ values[i] = value[1:-1]
+ return values
diff --git a/rattail/filemon/linux.py b/rattail/filemon/linux.py
index b71e35e73b42974f7ab34b880dfa99d631418f12..7d9100b7a6816aac2b5713915678b86192a6801b 100644
--- a/rattail/filemon/linux.py
+++ b/rattail/filemon/linux.py
@@ -1,9 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
################################################################################
#
# Rattail -- Retail Software Framework
-# Copyright © 2010-2012 Lance Edgar
+# Copyright © 2010-2014 Lance Edgar
#
# This file is part of Rattail.
#
@@ -23,29 +22,19 @@
################################################################################
"""
-``rattail.filemon.linux`` -- File Monitor for Linux
+File Monitor for Linux
"""
-import sys
-import os.path
-import threading
import Queue
import logging
-try:
- import pyinotify
-except ImportError:
- # Mock out for Windows.
- class Dummy(object):
- pass
- pyinotify = Dummy()
- pyinotify.ProcessEvent = Dummy
-
-import edbob
-from edbob.errors import email_exception
+import pyinotify
from rattail.daemon import Daemon
-from rattail import filemon
+from rattail.threads import Thread
+from rattail.filemon.config import load_profiles
+from rattail.filemon.actions import perform_actions
+from rattail.filemon.util import queue_existing
log = logging.getLogger(__name__)
@@ -53,46 +42,50 @@ log = logging.getLogger(__name__)
class EventHandler(pyinotify.ProcessEvent):
"""
- Event processor for file monitor daemon.
+ Event processor for file monitor daemon. This receives notifications of
+ file system events, and places new files on the queue as appropriate.
"""
def my_init(self, profile=None, **kwargs):
self.profile = profile
def process_IN_ACCESS(self, event):
- log.debug("EventHandler: IN_ACCESS: %s" % event.pathname)
+ log.debug(u"IN_ACCESS: {0}".format(event.pathname))
def process_IN_ATTRIB(self, event):
- log.debug("EventHandler: IN_ATTRIB: %s" % event.pathname)
+ log.debug(u"IN_ATTRIB: {0}".format(event.pathname))
def process_IN_CLOSE_WRITE(self, event):
- log.debug("EventHandler: IN_CLOSE_WRITE: %s" % event.pathname)
- if not self.profile.locks:
+ log.debug(u"IN_CLOSE_WRITE: {0}".format(event.pathname))
+ if not self.profile.watch_locks:
self.profile.queue.put(event.pathname)
def process_IN_CREATE(self, event):
- log.debug("EventHandler: IN_CREATE: %s" % event.pathname)
+ log.debug(u"IN_CREATE: {0}".format(event.pathname))
def process_IN_DELETE(self, event):
- log.debug("EventHandler: IN_DELETE: %s" % event.pathname)
- if self.profile.locks and event.pathname.endswith('.lock'):
+ log.debug(u"IN_DELETE: {0}".format(event.pathname))
+ if self.profile.watch_locks and event.pathname.endswith(u'.lock'):
self.profile.queue.put(event.pathname[:-5])
def process_IN_MODIFY(self, event):
- log.debug("EventHandler: IN_MODIFY: %s" % event.pathname)
+ log.debug(u"IN_MODIFY: {0}".format(event.pathname))
def process_IN_MOVED_TO(self, event):
- log.debug("EventHandler: IN_MOVED_TO: %s" % event.pathname)
- if not self.profile.locks:
+ log.debug(u"IN_MOVED_TO: {0}".format(event.pathname))
+ if not self.profile.watch_locks:
self.profile.queue.put(event.pathname)
class FileMonitorDaemon(Daemon):
+ """
+ Linux daemon implementation of the File Monitor.
+ """
def run(self):
- wm = pyinotify.WatchManager()
- notifier = pyinotify.Notifier(wm)
+ watch_manager = pyinotify.WatchManager()
+ notifier = pyinotify.Notifier(watch_manager)
mask = (pyinotify.IN_ACCESS
| pyinotify.IN_ATTRIB
@@ -102,7 +95,7 @@ class FileMonitorDaemon(Daemon):
| pyinotify.IN_MODIFY
| pyinotify.IN_MOVED_TO)
- monitored = filemon.get_monitor_profiles(self.config)
+ monitored = load_profiles(self.config)
for key, profile in monitored.iteritems():
# Create a file queue for the profile.
@@ -113,17 +106,16 @@ class FileMonitorDaemon(Daemon):
# Maybe put all pre-existing files in the queue.
if profile.process_existing:
- filemon.queue_existing(profile, path)
+ queue_existing(profile, path)
# Create a watch for the folder.
- log.debug("start_daemon: profile '%s' watches folder: %s" % (key, path))
- wm.add_watch(path, mask, proc_fun=EventHandler(profile=profile))
+ log.debug(u"adding watch to profile '{0}' for folder: {1}".format(key, path))
+ watch_manager.add_watch(path, mask, proc_fun=EventHandler(profile=profile))
# Create an action thread for the profile.
- name = 'actions-%s' % key
- log.debug("FileMonitorDaemon.run: starting action thread: %s" % name)
- thread = threading.Thread(target=filemon.perform_actions,
- name=name, args=(profile,))
+ name = u'actions-{0}'.format(key)
+ log.debug(u"starting action thread: {0}".format(name))
+ thread = Thread(target=perform_actions, name=name, args=(profile,))
thread.daemon = True
thread.start()
@@ -135,20 +127,16 @@ def get_daemon(config, pidfile=None):
"""
Get a :class:`FileMonitorDaemon` instance.
"""
-
if pidfile is None:
pidfile = config.get('rattail.filemon', 'pid_path',
default='/var/run/rattail/filemon.pid')
- daemon = FileMonitorDaemon(pidfile)
- daemon.config = config
- return daemon
+ return FileMonitorDaemon(pidfile, config=config)
def start_daemon(config, pidfile=None, daemonize=True):
"""
Start the file monitor daemon.
"""
-
get_daemon(config, pidfile).start(daemonize)
@@ -156,5 +144,4 @@ def stop_daemon(config, pidfile=None):
"""
Stop the file monitor daemon.
"""
-
get_daemon(config, pidfile).stop()
diff --git a/rattail/filemon/util.py b/rattail/filemon/util.py
index e14463ae37612079b0658703802a288f8b4bbb7a..c98e2d166d5a6f4326d76295e9db1ba3d3247ae2 100644
--- a/rattail/filemon/util.py
+++ b/rattail/filemon/util.py
@@ -1,9 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
################################################################################
#
# Rattail -- Retail Software Framework
-# Copyright © 2010-2012 Lance Edgar
+# Copyright © 2010-2014 Lance Edgar
#
# This file is part of Rattail.
#
@@ -26,15 +25,40 @@
File Monitor Utilities
"""
+import os
+import logging
-def raise_exception(path, message=u"Fake error for testing"):
+
+log = logging.getLogger(__name__)
+
+
+def queue_existing(profile, path):
"""
- File monitor action which always raises an exception.
+ Adds files found in a watched folder to a processing queue. This is called
+ when the monitor first starts, to handle the case of files which exist
+ prior to startup.
+
+ If files are found, they are first sorted by modification timestamp, using
+ a lexical sort on the filename as a tie-breaker, and then added to the
+ queue in that order.
- This is meant to be a simple way to test the error handling of a file
- monitor. For example, whether or not file processing continues for
- subsequent files after the first error is encountered. If logging
- configuration dictates that an email should be sent, it will of course test
- that as well.
+ :param profile: Monitor configuration profile for which the folder is to be
+ watched. The profile is expected to already have a queue attached; any
+ existing files will be added to this queue.
+
+ :param path: Folder path which is to be checked for files.
"""
- raise Exception(u'{0}: {1}'.format(message, path))
+ paths = [os.path.join(path, p) for p in os.listdir(path)]
+ paths = sorted(paths, key=lambda p: (os.path.getmtime(p), p))
+ for path in paths:
+
+ # Only process normal files.
+ if not os.path.isfile(path):
+ continue
+
+ # If using locks, don't process "in transit" files.
+ if profile.watch_locks and path.endswith(u'.lock'):
+ continue
+
+ log.debug(u"queuing existing file: {0}".format(path))
+ profile.queue.put(path)
diff --git a/rattail/filemon/win32.py b/rattail/filemon/win32.py
index 1d5019f2221d9fb955ee83af8915d4708d7b5394..ea7572107d571b913c2b2c837965a6bd2921b3c2 100644
--- a/rattail/filemon/win32.py
+++ b/rattail/filemon/win32.py
@@ -1,9 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
################################################################################
#
# Rattail -- Retail Software Framework
-# Copyright © 2010-2012 Lance Edgar
+# Copyright © 2010-2014 Lance Edgar
#
# This file is part of Rattail.
#
@@ -23,37 +22,43 @@
################################################################################
"""
-``rattail.filemon.win32`` -- File Monitor for Windows
+File Monitor for Windows
"""
-import os.path
+import os
import Queue
import logging
-from rattail import filemon
from rattail.win32.service import Service
from rattail.threads import Thread
+from rattail.filemon.config import load_profiles
+from rattail.filemon.actions import perform_actions
+from rattail.filemon.util import queue_existing
+# TODO: Would be nice to have a note explaining why this hack exists.
name = __name__
-if name == 'win32':
- name = 'rattail.filemon.win32'
+if name == u'win32':
+ name = u'rattail.filemon.win32'
log = logging.getLogger(name)
class RattailFileMonitor(Service):
+ """
+ Windows service implementation of the File Monitor.
+ """
- _svc_name_ = "RattailFileMonitor"
- _svc_display_name_ = "Rattail : File Monitoring Service"
- _svc_description_ = ("Monitors one or more folders for incoming files, "
- "and performs configured actions as new files arrive.")
+ _svc_name_ = u"RattailFileMonitor"
+ _svc_display_name_ = u"Rattail : File Monitoring Service"
+ _svc_description_ = (u"Monitors one or more folders for incoming files, "
+ u"and performs configured actions as new files arrive.")
def Initialize(self, config):
"""
Service initialization.
"""
# Read monitor profile(s) from config.
- self.monitored = filemon.get_monitor_profiles(config)
+ self.monitored = load_profiles(config)
# Make sure we have something to do.
if not self.monitored:
@@ -70,28 +75,28 @@ class RattailFileMonitor(Service):
# Maybe put all pre-existing files in the queue.
if profile.process_existing:
- filemon.queue_existing(profile, path)
+ queue_existing(profile, path)
- # Create a monitor thread for the folder.
- name = u'monitor_{0}-{1}'.format(key, i)
- log.debug(u"starting '{0}' thread for folder: {1}".format(name, path))
- thread = Thread(target=monitor_files, name=name, args=(profile, path))
+ # Create a watcher thread for the folder.
+ name = u'watcher_{0}-{1}'.format(key, i)
+ log.debug(u"starting {0} thread for folder: {1}".format(repr(name), repr(path)))
+ thread = Thread(target=watch_directory, name=name, args=(profile, path))
thread.daemon = True
thread.start()
# Create an action thread for the profile.
name = u'actions_{0}'.format(key)
- log.debug(u"starting action thread: {0}".format(name))
- thread = Thread(target=filemon.perform_actions, name=name, args=(profile,))
+ log.debug(u"starting action thread: {0}".format(repr(name)))
+ thread = Thread(target=perform_actions, name=name, args=(profile,))
thread.daemon = True
thread.start()
return True
-def monitor_files(profile, path):
+def watch_directory(profile, path):
"""
- Callable target for file monitor threads.
+ Callable target for watcher threads.
"""
import win32file
@@ -122,7 +127,7 @@ def monitor_files(profile, path):
for action, fname in results:
fpath = os.path.join(path, fname)
queue = False
- if profile.locks:
+ if profile.watch_locks:
if action == winnt.FILE_ACTION_REMOVED and fpath.endswith('.lock'):
fpath = fpath[:-5]
queue = True
diff --git a/tests/filemon/test_actions.py b/tests/filemon/test_actions.py
new file mode 100644
index 0000000000000000000000000000000000000000..d53a798d43084839a82ad353e4d3aaef198435a8
--- /dev/null
+++ b/tests/filemon/test_actions.py
@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+
+import os
+import time
+import Queue
+from unittest import TestCase
+
+from mock import Mock, patch, call
+from fixture import TempIO
+
+from edbob.configuration import AppConfigParser
+
+from rattail.filemon import actions
+from rattail.filemon.config import Profile, ProfileAction
+
+
+class TestAction(TestCase):
+
+ def test_callable_must_be_implemented_in_subclass(self):
+ config = AppConfigParser(u'rattail')
+ action = actions.Action(config)
+ self.assertRaises(NotImplementedError, action)
+
+
+@patch(u'rattail.filemon.actions.noop')
+class TestPerformActions(TestCase):
+
+ def setUp(self):
+ self.tmp = TempIO()
+ self.config = AppConfigParser(u'rattail')
+ self.config.set(u'rattail.filemon', u'monitor', u'foo')
+ self.config.set(u'rattail.filemon', u'foo.dirs', self.tmp)
+ self.config.set(u'rattail.filemon', u'foo.actions', u'noop')
+ self.config.set(u'rattail.filemon', u'foo.action.noop.func', u'rattail.filemon.actions:noop')
+ # Must delay creating the profile since doing it here would bypass our mock of noop.
+
+ def get_profile(self, stop_on_error=False):
+ profile = Profile(self.config, u'foo')
+ profile.stop_on_error = stop_on_error
+ profile.queue = Mock()
+ profile.queue.get_nowait.side_effect = [
+ Queue.Empty, # for coverage sake; will be effectively skipped
+ self.tmp.putfile(u'file1', u''),
+ self.tmp.putfile(u'file2', u''),
+ self.tmp.putfile(u'file3', u''),
+ actions.StopProcessing,
+ ]
+ return profile
+
+ def test_action_is_invoked_for_each_file_in_queue(self, noop):
+ profile = self.get_profile()
+ actions.perform_actions(profile)
+ self.assertEqual(noop.call_count, 3)
+ noop.assert_has_calls([
+ call(self.tmp.join(u'file1')),
+ call(self.tmp.join(u'file2')),
+ call(self.tmp.join(u'file3')),
+ ])
+
+ def test_action_is_skipped_for_nonexistent_file(self, noop):
+ profile = self.get_profile()
+ os.remove(self.tmp.join(u'file2'))
+ actions.perform_actions(profile)
+ self.assertEqual(noop.call_count, 2)
+ # no call for file2
+ noop.assert_has_calls([
+ call(self.tmp.join(u'file1')),
+ call(self.tmp.join(u'file3')),
+ ])
+
+ def test_action_which_raises_error_causes_subsequent_actions_to_be_skipped_for_same_file(self, noop):
+ self.config.set(u'rattail.filemon', u'foo.actions', u'noop, delete')
+ self.config.set(u'rattail.filemon', u'foo.action.delete.func', u'os:remove')
+ profile = self.get_profile()
+ # processing second file fails, so it shouldn't be deleted
+ noop.side_effect = [None, RuntimeError, None]
+ actions.perform_actions(profile)
+ self.assertFalse(os.path.exists(self.tmp.join(u'file1')))
+ self.assertTrue(os.path.exists(self.tmp.join(u'file2')))
+ self.assertFalse(os.path.exists(self.tmp.join(u'file3')))
+
+ def test_action_which_raises_error_causes_all_processing_to_stop_if_so_configured(self, noop):
+ self.config.set(u'rattail.filemon', u'foo.actions', u'noop, delete')
+ self.config.set(u'rattail.filemon', u'foo.action.delete.func', u'os:remove')
+ profile = self.get_profile(stop_on_error=True)
+ # processing second file fails; third file shouldn't be processed at all
+ noop.side_effect = [None, RuntimeError, None]
+ actions.perform_actions(profile)
+ self.assertEqual(noop.call_count, 2)
+ noop.assert_has_calls([
+ call(self.tmp.join(u'file1')),
+ call(self.tmp.join(u'file2')),
+ ])
+ self.assertFalse(os.path.exists(self.tmp.join(u'file1')))
+ self.assertTrue(os.path.exists(self.tmp.join(u'file2')))
+ self.assertTrue(os.path.exists(self.tmp.join(u'file3')))
+
+
+class TestInvokeAction(TestCase):
+
+ def setUp(self):
+ self.action = ProfileAction()
+ self.action.action = Mock(return_value=None)
+ self.action.retry_attempts = 6
+ self.tmp = TempIO()
+ self.file = self.tmp.putfile(u'file', u'')
+
+ def test_action_which_succeeds_is_only_called_once(self):
+ actions.invoke_action(self.action, self.file)
+ self.assertEqual(self.action.action.call_count, 1)
+
+ def test_action_with_no_delay_does_not_pause_between_attempts(self):
+ self.action.retry_attempts = 3
+ self.action.action.side_effect = [RuntimeError, RuntimeError, None]
+ start = time.time()
+ actions.invoke_action(self.action, self.file)
+ self.assertEqual(self.action.action.call_count, 3)
+ self.assertTrue(time.time() - start < 1.0)
+
+ def test_action_with_delay_pauses_between_attempts(self):
+ self.action.retry_attempts = 3
+ self.action.retry_delay = 1
+ self.action.action.side_effect = [RuntimeError, RuntimeError, None]
+ start = time.time()
+ actions.invoke_action(self.action, self.file)
+ self.assertEqual(self.action.action.call_count, 3)
+ self.assertTrue(time.time() - start >= 2.0)
+
+ def test_action_which_fails_is_only_attempted_the_specified_number_of_times(self):
+ self.action.action.side_effect = RuntimeError
+ # Last attempt will not handle the exception; assert that as well.
+ self.assertRaises(RuntimeError, actions.invoke_action, self.action, self.file)
+ self.assertEqual(self.action.action.call_count, 6)
+
+ def test_action_which_fails_then_succeeds_stops_retrying(self):
+ # First 2 attempts fail, third succeeds.
+ self.action.action.side_effect = [RuntimeError, RuntimeError, None]
+ actions.invoke_action(self.action, self.file)
+ self.assertEqual(self.action.action.call_count, 3)
+
+ def test_action_which_fails_with_different_errors_stops_retrying(self):
+ self.action.action.side_effect = [ValueError, TypeError, None]
+ # Second attempt will not handle the exception; assert that as well.
+ self.assertRaises(TypeError, actions.invoke_action, self.action, self.file)
+ self.assertEqual(self.action.action.call_count, 2)
+
+
+class TestRaiseException(TestCase):
+
+ def test_exception_is_raised(self):
+ # this hardly deserves a test, but what the hell
+ self.assertRaises(Exception, actions.raise_exception, '/dev/null')
diff --git a/tests/filemon/test_config.py b/tests/filemon/test_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..1434fe034314d8e8b6a7fcd3ec536377884f6a27
--- /dev/null
+++ b/tests/filemon/test_config.py
@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+
+import os
+from unittest import TestCase
+
+from fixture import TempIO
+
+from edbob.configuration import AppConfigParser
+
+from rattail.filemon import config
+from rattail.filemon import Action
+from rattail.exceptions import ConfigurationError
+
+
+class TestProfile(TestCase):
+
+ def setUp(self):
+ self.config = AppConfigParser(u'rattail')
+ self.config.set(u'rattail.filemon', u'foo.actions', u'bar')
+
+ def test_empty_config_means_empty_profile(self):
+ profile = config.Profile(self.config, u'nonexistent_key')
+ self.assertEqual(len(profile.dirs), 0)
+ self.assertFalse(profile.watch_locks)
+ self.assertTrue(profile.process_existing)
+ self.assertFalse(profile.stop_on_error)
+ self.assertEqual(len(profile.actions), 0)
+
+ def test_action_must_specify_callable(self):
+ self.assertRaises(ConfigurationError, config.Profile, self.config, u'foo')
+
+ def test_action_must_not_specify_both_func_and_class_callables(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.class', u'baz')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'baz')
+ self.assertRaises(ConfigurationError, config.Profile, self.config, u'foo')
+
+ def test_action_with_func_callable(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'os:remove')
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ action = profile.actions[0]
+ self.assertEqual(action.spec, u'os:remove')
+ self.assertTrue(action.action is os.remove)
+
+ def test_action_with_class_callable(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.class', u'rattail.filemon:Action')
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ action = profile.actions[0]
+ self.assertEqual(action.spec, u'rattail.filemon:Action')
+ self.assertTrue(isinstance(action.action, Action))
+
+ def test_action_with_args(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'shutil:move')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.args', u'/dev/null')
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ action = profile.actions[0]
+ self.assertEqual(len(action.args), 1)
+ self.assertEqual(action.args[0], u'/dev/null')
+
+ def test_action_with_kwargs(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:raise_exception')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.kwarg.message', u"Hello World")
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ action = profile.actions[0]
+ self.assertEqual(len(action.kwargs), 1)
+ self.assertEqual(action.kwargs[u'message'], u"Hello World")
+
+ def test_action_with_default_retry(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:noop')
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ action = profile.actions[0]
+ self.assertEqual(action.retry_attempts, 1)
+ self.assertEqual(action.retry_delay, 0)
+
+ def test_action_with_valid_configured_retry(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:noop')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.retry_attempts', u'42')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.retry_delay', u'100')
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ action = profile.actions[0]
+ self.assertEqual(action.retry_attempts, 42)
+ self.assertEqual(action.retry_delay, 100)
+
+ def test_action_with_invalid_configured_retry(self):
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'rattail.filemon.actions:noop')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.retry_attempts', u'-1')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.retry_delay', u'-1')
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ action = profile.actions[0]
+ self.assertEqual(action.retry_attempts, 1)
+ self.assertEqual(action.retry_delay, 0)
+
+ def test_normalize_dirs(self):
+ tmp = TempIO()
+ dir1 = tmp.mkdir(u'dir1')
+ # dir2 will be pruned due to its not existing
+ dir2 = tmp.mkdir(u'dir2')
+ os.rmdir(dir2)
+ # file1 will be pruned due to its not being a directory
+ file1 = tmp.putfile(u'file1', u'')
+ self.config.set(u'rattail.filemon', u'foo.action.bar.func', u'os:remove')
+ self.config.set(u'rattail.filemon', u'foo.dirs', u' '.join([u'"{0}"'.format(d) for d in [dir1, dir2, file1]]))
+ profile = config.Profile(self.config, u'foo')
+ self.assertEqual(len(profile.dirs), 1)
+ self.assertEqual(profile.dirs[0], dir1)
+
+
+class TestLoadProfiles(TestCase):
+
+ def setUp(self):
+ self.tmp = TempIO()
+ self.config = AppConfigParser(u'rattail')
+ self.config.set(u'rattail.filemon', u'monitor', u'foo, bar')
+ self.config.set(u'rattail.filemon', u'foo.dirs', u'"{0}"'.format(self.tmp))
+ self.config.set(u'rattail.filemon', u'foo.actions', u'delete')
+ self.config.set(u'rattail.filemon', u'foo.action.delete.func', u'os:remove')
+ self.config.set(u'rattail.filemon', u'bar.dirs', u'"{0}"'.format(self.tmp))
+ self.config.set(u'rattail.filemon', u'bar.actions', u'delete')
+ self.config.set(u'rattail.filemon', u'bar.action.delete.func', u'os:remove')
+
+ def test_returns_all_profiles_specified_in_monitor_option(self):
+ monitored = config.load_profiles(self.config)
+ self.assertEqual(len(monitored), 2)
+ # leave profiles intact but replace monitor option with one key only
+ self.config.set(u'rattail.filemon', u'monitor', u'foo')
+ monitored = config.load_profiles(self.config)
+ self.assertEqual(len(monitored), 1)
+
+ def test_monitor_option_must_be_specified(self):
+ self.config.remove_option(u'rattail.filemon', u'monitor')
+ self.assertRaises(ConfigurationError, config.load_profiles, self.config)
+
+ def test_profiles_which_define_no_watched_folders_are_pruned(self):
+ monitored = config.load_profiles(self.config)
+ self.assertEqual(len(monitored), 2)
+ # remove foo's watched folder(s)
+ self.config.remove_option(u'rattail.filemon', u'foo.dirs')
+ monitored = config.load_profiles(self.config)
+ self.assertEqual(len(monitored), 1)
+
+ def test_profiles_which_define_no_actions_are_pruned(self):
+ monitored = config.load_profiles(self.config)
+ self.assertEqual(len(monitored), 2)
+ # remove foo's actions
+ self.config.remove_option(u'rattail.filemon', u'foo.actions')
+ monitored = config.load_profiles(self.config)
+ self.assertEqual(len(monitored), 1)
+
+ def test_fallback_to_legacy_mode(self):
+ # replace 'monitor' option with 'monitored' and update profiles accordingly
+ self.config.remove_option(u'rattail.filemon', u'monitor')
+ self.config.set(u'rattail.filemon', u'monitored', u'foo, bar')
+ self.config.set(u'rattail.filemon', u'foo.dirs', u"['{0}']".format(self.tmp))
+ self.config.set(u'rattail.filemon', u'foo.actions', u"['os:remove']")
+ self.config.set(u'rattail.filemon', u'bar.dirs', u"['{0}']".format(self.tmp))
+ self.config.set(u'rattail.filemon', u'bar.actions', u"['os:remove']")
+ monitored = config.load_profiles(self.config)
+ self.assertEqual(len(monitored), 2)
+ profiles = list(monitored.values())
+ self.assertTrue(isinstance(profiles[0], config.LegacyProfile))
+ self.assertTrue(isinstance(profiles[1], config.LegacyProfile))
+
+
+class TestLegacyProfile(TestCase):
+
+ def setUp(self):
+ self.config = AppConfigParser(u'rattail')
+
+ def test_empty_config_means_empty_profile(self):
+ profile = config.LegacyProfile(self.config, u'nonexistent_key')
+ self.assertEqual(len(profile.dirs), 0)
+ self.assertFalse(profile.watch_locks)
+ self.assertTrue(profile.process_existing)
+ self.assertFalse(profile.stop_on_error)
+ self.assertEqual(len(profile.actions), 0)
+
+ def test_action_with_spec_only(self):
+ self.config.set(u'rattail.filemon', u'foo.actions', u"['os:remove']")
+ profile = config.LegacyProfile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ spec, action, args, kwargs = profile.actions[0]
+ self.assertEqual(spec, u'os:remove')
+ self.assertTrue(action is os.remove)
+
+ def test_action_with_spec_and_args(self):
+ self.config.set(u'rattail.filemon', u'foo.actions', u"[('shutil:move', u'/dev/null')]")
+ profile = config.LegacyProfile(self.config, u'foo')
+ self.assertEqual(len(profile.actions), 1)
+ spec, action, args, kwargs = profile.actions[0]
+ self.assertEqual(spec, u'shutil:move')
+ self.assertEqual(len(args), 1)
+ self.assertEqual(args[0], u'/dev/null')
+
+ def test_normalize_dirs(self):
+ tmp = TempIO()
+ dir1 = tmp.mkdir(u'dir1')
+ # dir2 will be pruned due to its not existing
+ dir2 = tmp.mkdir(u'dir2')
+ os.rmdir(dir2)
+ # file1 will be pruned due to its not being a directory
+ file1 = tmp.putfile(u'file1', u'')
+ self.config.set(u'rattail.filemon', u'foo.dirs', u"[{0}]".format(u', '.join([u"'{0}'".format(d) for d in [dir1, dir2, file1]])))
+ profile = config.LegacyProfile(self.config, u'foo')
+ self.assertEqual(len(profile.dirs), 1)
+ self.assertEqual(profile.dirs[0], dir1)
+
+
+class TestLoadLegacyProfiles(TestCase):
+
+ def setUp(self):
+ self.tmp = TempIO()
+ self.config = AppConfigParser(u'rattail')
+ self.config.set(u'rattail.filemon', u'monitored', u'foo, bar')
+ self.config.set(u'rattail.filemon', u'foo.dirs', u"['{0}']".format(self.tmp))
+ self.config.set(u'rattail.filemon', u'foo.actions', u"['os:remove']")
+ self.config.set(u'rattail.filemon', u'bar.dirs', u"['{0}']".format(self.tmp))
+ self.config.set(u'rattail.filemon', u'bar.actions', u"['os:remove']")
+
+ def test_returns_all_profiles_specified_in_monitor_option(self):
+ monitored = config.load_legacy_profiles(self.config)
+ self.assertEqual(len(monitored), 2)
+ # leave profiles intact but replace monitored option with one key only
+ self.config.set(u'rattail.filemon', u'monitored', u'foo')
+ monitored = config.load_legacy_profiles(self.config)
+ self.assertEqual(len(monitored), 1)
+
+ def test_monitor_option_must_be_specified(self):
+ self.config.remove_option(u'rattail.filemon', u'monitored')
+ self.assertRaises(ConfigurationError, config.load_legacy_profiles, self.config)
+
+ def test_profiles_which_define_no_watched_folders_are_pruned(self):
+ monitored = config.load_legacy_profiles(self.config)
+ self.assertEqual(len(monitored), 2)
+ # remove foo's watched folder(s)
+ self.config.remove_option(u'rattail.filemon', u'foo.dirs')
+ monitored = config.load_legacy_profiles(self.config)
+ self.assertEqual(len(monitored), 1)
+
+ def test_profiles_which_define_no_actions_are_pruned(self):
+ monitored = config.load_legacy_profiles(self.config)
+ self.assertEqual(len(monitored), 2)
+ # remove foo's actions
+ self.config.remove_option(u'rattail.filemon', u'foo.actions')
+ monitored = config.load_legacy_profiles(self.config)
+ self.assertEqual(len(monitored), 1)
+
+
+class TestParseList(TestCase):
+
+ def test_none(self):
+ value = config.parse_list(None)
+ self.assertEqual(len(value), 0)
+
+ def test_single_value(self):
+ value = config.parse_list(u'foo')
+ self.assertEqual(len(value), 1)
+ self.assertEqual(value[0], u'foo')
+
+ def test_single_value_padded_by_spaces(self):
+ value = config.parse_list(u' foo ')
+ self.assertEqual(len(value), 1)
+ self.assertEqual(value[0], u'foo')
+
+ def test_slash_is_not_a_separator(self):
+ value = config.parse_list(u'/dev/null')
+ self.assertEqual(len(value), 1)
+ self.assertEqual(value[0], u'/dev/null')
+
+ def test_multiple_values_separated_by_whitespace(self):
+ value = config.parse_list(u'foo bar baz')
+ self.assertEqual(len(value), 3)
+ self.assertEqual(value[0], u'foo')
+ self.assertEqual(value[1], u'bar')
+ self.assertEqual(value[2], u'baz')
+
+ def test_multiple_values_separated_by_commas(self):
+ value = config.parse_list(u'foo,bar,baz')
+ self.assertEqual(len(value), 3)
+ self.assertEqual(value[0], u'foo')
+ self.assertEqual(value[1], u'bar')
+ self.assertEqual(value[2], u'baz')
+
+ def test_multiple_values_separated_by_whitespace_and_commas(self):
+ value = config.parse_list(u' foo, bar baz')
+ self.assertEqual(len(value), 3)
+ self.assertEqual(value[0], u'foo')
+ self.assertEqual(value[1], u'bar')
+ self.assertEqual(value[2], u'baz')
+
+ def test_multiple_values_separated_by_whitespace_and_commas_with_some_quoting(self):
+ value = config.parse_list(u"""
+ foo
+ "C:\\some path\\with spaces\\and, a comma",
+ baz
+""")
+ self.assertEqual(len(value), 3)
+ self.assertEqual(value[0], u'foo')
+ self.assertEqual(value[1], u'C:\\some path\\with spaces\\and, a comma')
+ self.assertEqual(value[2], u'baz')
diff --git a/tests/filemon/test_linux.py b/tests/filemon/test_linux.py
new file mode 100644
index 0000000000000000000000000000000000000000..7bcf31a162f98cb6c8bba8e95d56d8d50f8056eb
--- /dev/null
+++ b/tests/filemon/test_linux.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+import Queue
+from unittest import TestCase
+
+from mock import Mock
+from fixture import TempIO
+
+from edbob.configuration import AppConfigParser
+
+from rattail.filemon import linux
+from rattail.filemon.config import Profile
+
+
+class TestEventHandler(TestCase):
+
+ def setUp(self):
+ self.tmp = TempIO()
+ self.config = AppConfigParser(u'rattail')
+ self.config.set(u'rattail.filemon', u'monitor', u'foo')
+ self.config.set(u'rattail.filemon', u'foo.dirs', self.tmp)
+ self.config.set(u'rattail.filemon', u'foo.actions', u'noop')
+ self.config.set(u'rattail.filemon', u'foo.action.noop.func', u'rattail.filemon.actions:noop')
+ self.profile = Profile(self.config, u'foo')
+ self.profile.queue = Queue.Queue()
+ self.handler = linux.EventHandler()
+ self.handler.my_init(self.profile)
+
+ def test_in_access_event_does_nothing(self):
+ event = Mock(pathname=self.tmp.putfile(u'file', u''))
+ self.handler.process_IN_ACCESS(event)
+ self.assertTrue(self.profile.queue.empty())
+
+ def test_in_attrib_event_does_nothing(self):
+ event = Mock(pathname=self.tmp.putfile(u'file', u''))
+ self.handler.process_IN_ATTRIB(event)
+ self.assertTrue(self.profile.queue.empty())
+
+ def test_in_create_event_does_nothing(self):
+ event = Mock(pathname=self.tmp.putfile(u'file', u''))
+ self.handler.process_IN_CREATE(event)
+ self.assertTrue(self.profile.queue.empty())
+
+ def test_in_modify_event_does_nothing(self):
+ event = Mock(pathname=self.tmp.putfile(u'file', u''))
+ self.handler.process_IN_MODIFY(event)
+ self.assertTrue(self.profile.queue.empty())
+
+ def test_in_close_write_event_queues_file_if_profile_does_not_watch_locks(self):
+ event = Mock(pathname=self.tmp.putfile(u'file', u''))
+ self.profile.watch_locks = False
+ self.handler.process_IN_CLOSE_WRITE(event)
+ self.assertEqual(self.profile.queue.qsize(), 1)
+ self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file'))
+
+ def test_in_close_write_event_does_nothing_if_profile_watches_locks(self):
+ event = Mock(pathname=self.tmp.putfile(u'file.lock', u''))
+ self.profile.watch_locks = True
+ self.handler.process_IN_CLOSE_WRITE(event)
+ self.assertTrue(self.profile.queue.empty())
+
+ def test_in_moved_to_event_queues_file_if_profile_does_not_watch_locks(self):
+ event = Mock(pathname=self.tmp.putfile(u'file', u''))
+ self.profile.watch_locks = False
+ self.handler.process_IN_MOVED_TO(event)
+ self.assertEqual(self.profile.queue.qsize(), 1)
+ self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file'))
+
+ def test_in_moved_to_event_does_nothing_if_profile_watches_locks(self):
+ event = Mock(pathname=self.tmp.putfile(u'file.lock', u''))
+ self.profile.watch_locks = True
+ self.handler.process_IN_MOVED_TO(event)
+ self.assertTrue(self.profile.queue.empty())
+
+ def test_in_delete_event_queues_file_if_profile_watches_locks(self):
+ event = Mock(pathname=self.tmp.putfile(u'file.lock', u''))
+ self.profile.watch_locks = True
+ self.handler.process_IN_DELETE(event)
+ self.assertEqual(self.profile.queue.qsize(), 1)
+ self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file'))
+
+ def test_in_moved_to_event_does_nothing_if_profile_does_not_watch_locks(self):
+ event = Mock(pathname=self.tmp.putfile(u'file', u''))
+ self.profile.watch_locks = False
+ self.handler.process_IN_DELETE(event)
+ self.assertTrue(self.profile.queue.empty())
diff --git a/tests/filemon/test_util.py b/tests/filemon/test_util.py
index f9ed406539265459e2e0232a6e0a6894efe0357a..c0b9e211c3f0c12998b48663686403b247ac2f6d 100644
--- a/tests/filemon/test_util.py
+++ b/tests/filemon/test_util.py
@@ -1,15 +1,45 @@
+# -*- coding: utf-8 -*-
+import Queue
from unittest import TestCase
+from fixture import TempIO
+
+from edbob.configuration import AppConfigParser
+
from rattail.filemon import util
+from rattail.filemon.config import Profile
+
+
+class TestQueueExisting(TestCase):
+
+ def setUp(self):
+ self.tmp = TempIO()
+ self.config = AppConfigParser(u'rattail')
+ self.config.set(u'rattail.filemon', u'monitor', u'foo')
+ self.config.set(u'rattail.filemon', u'foo.dirs', self.tmp)
+ self.config.set(u'rattail.filemon', u'foo.actions', u'noop')
+ self.config.set(u'rattail.filemon', u'foo.action.noop.func', u'rattail.filemon.actions:noop')
+ self.profile = Profile(self.config, u'foo')
+ self.profile.queue = Queue.Queue()
+ def test_nothing_queued_if_no_files_exist(self):
+ util.queue_existing(self.profile, self.tmp)
+ self.assertTrue(self.profile.queue.empty())
-class TestRaiseException(TestCase):
+ def test_normal_files_are_queued_but_not_folders(self):
+ self.tmp.putfile(u'file', u'')
+ self.tmp.mkdir(u'folder')
+ util.queue_existing(self.profile, self.tmp)
+ self.assertEqual(self.profile.queue.qsize(), 1)
+ self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file'))
+ self.assertTrue(self.profile.queue.empty())
- def test_exception_is_raised_and_has_correct_message(self):
- path = u'fake_path'
- self.assertRaises(Exception, util.raise_exception, path)
- try:
- util.raise_exception(path, message=u"This is my custom message")
- except Exception as error:
- self.assertEqual(unicode(error), u"This is my custom message: fake_path")
+ def test_if_profile_watches_locks_then_normal_files_are_queued_but_not_lock_files(self):
+ self.profile.watch_locks = True
+ self.tmp.putfile(u'file1.lock', u'')
+ self.tmp.putfile(u'file2', u'')
+ util.queue_existing(self.profile, self.tmp)
+ self.assertEqual(self.profile.queue.qsize(), 1)
+ self.assertEqual(self.profile.queue.get_nowait(), self.tmp.join(u'file2'))
+ self.assertTrue(self.profile.queue.empty())