docs/_build/

.. -*- coding: utf-8 -*-

``rattail.filemon.actions``

===========================

.. automodule:: rattail.filemon.actions

   :members:

.. -*- coding: utf-8 -*-

``rattail.filemon.config``

==========================

.. automodule:: rattail.filemon.config

   :members:

.. -*- coding: utf-8 -*-

``rattail.filemon``

===================

.. automodule:: rattail.filemon

.. autoclass:: Action

.. -*- coding: utf-8 -*-

``rattail.filemon.linux``

=========================

.. automodule:: rattail.filemon.linux

   :members:

.. -*- coding: utf-8 -*-

.. -*- coding: utf-8 -*-

``rattail.filemon.win32``

=========================

.. automodule:: rattail.filemon.win32

   :members:

   api/rattail/filemon/index

   api/rattail/filemon/actions

   api/rattail/filemon/config

   api/rattail/filemon/linux

   api/rattail/filemon/win32

#  Copyright © 2010-2014 Lance Edgar

class ConfigurationError(RattailError):

    """

    Generic class for configuration errors.

    """

#  Copyright © 2010-2014 Lance Edgar

File Monitor

from rattail.filemon.actions import Action

# -*- coding: utf-8 -*-

################################################################################

#

#  Rattail -- Retail Software Framework

#  Copyright © 2010-2014 Lance Edgar

#

#  This file is part of Rattail.

#

#  Rattail is free software: you can redistribute it and/or modify it under the

#  terms of the GNU Affero General Public License as published by the Free

#  Software Foundation, either version 3 of the License, or (at your option)

#  any later version.

#

#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY

#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for

#  more details.

#

#  You should have received a copy of the GNU Affero General Public License

#  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.

#

################################################################################

"""

File Monitor Actions

"""

import os

import sys

import time

import Queue

import logging

log = logging.getLogger(__name__)

class Action(object):

    """

    Base class for file monitor actions.

    """

    def __init__(self, config):

        self.config = config

    def __call__(self, *args, **kwargs):

        """

        This method must be implemented in the subclass; it defines what the

        action actually *does*.  The file monitor will invoke this method for

        all new files which are discovered.

        """

        raise NotImplementedError

class StopProcessing(Exception):

    """

    Simple exception to indicate action processing should stop.  This is really

    only useful for tests.

    """

def perform_actions(profile):

    """

    Target for action threads.  Provides the main loop which checks the queue

    for new files and invokes actions for each, as they appear.

    """

    # If running on Windows, we add a step to help ensure the file is truly

    # free of competing process interests.  (In fact it would be nice to have

    # this on Linux as well, but I'm not sure how to do it there.)

    wait_for_file = lambda p: p

    if sys.platform.startswith(u'win'): # pragma: no cover

        import win32api

        from rattail.win32 import file_is_free

        def wait_for_file(path):

            while not file_is_free(path):

                win32api.Sleep(0)

    stop = False

    while not stop:

        try:

            path = profile.queue.get_nowait()

        except Queue.Empty:

            pass

        except StopProcessing:

            stop = True

        else:

            log.debug(u"queue contained a file: {0}".format(repr(path)))

            # In some cases, processing one file may cause other related files

            # to also be processed.  When this happens, a path on the queue may

            # point to a file which no longer exists.

            if not os.path.exists(path):

                log.warning(u"file path does not exist: {0}".format(path))

                continue

            # This does nothing unless running on Windows.

            wait_for_file(path)

            for action in profile.actions:

                try:

                    invoke_action(action, path)

                except:

                    # Stop processing files altogether for this profile if it

                    # is so configured.

                    if profile.stop_on_error:

                        log.warning(u"an error was encountered, and configuration dictates that no more "

                                    u"actions will be processed for profile {0}".format(repr(profile.key)))

                        stop = True

                    # Either way no more actions should be invoked for this

                    # particular file.

                    break

def invoke_action(action, path):

    """

    Invoke a single action on a file, retrying as necessary.

    """

    attempts = 0

    errtype = None

    while True:

        attempts += 1

        log.debug(u"invoking action {0} (attempt #{1} of {2}) on file: {3}".format(

                repr(action.spec), attempts, action.retry_attempts, repr(path)))

        try:

            action.action(path, *action.args, **action.kwargs)

        except Exception as error:

            # If we've reached our final attempt, stop retrying.

            if attempts >= action.retry_attempts:

                log.exception(u"attempt #{0} failed for action {1} (giving up) on "

                              u"file: {2}".format(attempts, repr(action.spec), repr(path)))

                raise

            # If this exception is not the first, and is of a different type

            # than seen previously, do *not* continue to retry.

            if errtype is not None and not isinstance(error, errtype):

                log.exception(u"new exception differs from previous one(s), giving up on "

                              u"action {0} for file: {1}".format(repr(action.spec), repr(path)))

                raise

            # Record the type of exception seen, and pause for next retry.

            log.warning(u"attempt #{0} failed for action {1} on file: {2}".format(

                    attempts, repr(action.spec), repr(path)), exc_info=True)

            errtype = type(error)

            log.debug(u"pausing for {0} seconds before making attempt #{1} of {2}".format(

                    action.retry_delay, attempts + 1, action.retry_attempts))

            if action.retry_delay:

                time.sleep(action.retry_delay)

        else:

            # No error, invocation successful.

            log.debug(u"attempt #{0} succeeded for action {1} on file: {2}".format(

                    attempts, repr(action.spec), repr(path)))

            break

def raise_exception(path, message=u"Fake error for testing"):

    """

    File monitor action which always raises an exception.

    This is meant to be a simple way to test the error handling of a file

    monitor.  For example, whether or not file processing continues for

    subsequent files after the first error is encountered.  If logging

    configuration dictates that an email should be sent, it will of course test

    that as well.

    """

    raise Exception(u'{0}: {1}'.format(message, path))

def noop(path):

    """

    File monitor action which does nothing at all.

    This exists for the sake of tests.  I doubt it's useful in any other

    context.

    """

# -*- coding: utf-8 -*-

################################################################################

#

#  Rattail -- Retail Software Framework

#  Copyright © 2010-2014 Lance Edgar

#

#  This file is part of Rattail.

#

#  Rattail is free software: you can redistribute it and/or modify it under the

#  terms of the GNU Affero General Public License as published by the Free

#  Software Foundation, either version 3 of the License, or (at your option)

#  any later version.

#

#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY

#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

#  FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for

#  more details.

#

#  You should have received a copy of the GNU Affero General Public License

#  along with Rattail.  If not, see <http://www.gnu.org/licenses/>.

#

################################################################################

"""

File Monitor Configuration

"""

import os

import re

import shlex

import warnings

import logging

from rattail.util import load_object

from rattail.exceptions import ConfigurationError

log = logging.getLogger(__name__)

class ProfileAction(object):

    """

    Simple class to hold configuration for a particular action defined within a

    monitor :class:`Profile`.  Each instance has the following attributes:

    .. attribute:: spec

       The original "spec" string used to obtain the action callable.

    .. attribute:: action

       A reference to the action callable.

    .. attribute:: args

       A sequence of positional arguments to be passed to the callable (in

       addition to the file path) when invoking the action.

    .. attribute:: kwargs

       A dictionary of keyword arguments to be passed to the callable (in

       addition to the positional arguments) when invoking the action.

    .. attribute:: retry_attempts

       Number of attempts to make when invoking the action.  Defaults to ``1``,

       meaning the first attempt will be made but no retries will happen.

    .. attribute:: retry_delay

       Number of seconds to pause between retry attempts, if

       :attr:`retry_attempts` is greater than one.  Defaults to ``0``.

    """

    spec = None

    action = None

    args = []

    kwargs = {}

    retry_attempts = 1

    retry_delay = 0

class Profile(object):

    """

    Simple class to hold configuration for a file monitor "profile".  The

    profile determines which folders to watch, which actions to take when new

    files appear, etc.  Each instance of this class has the following

    attributes:

    .. attribute:: config

       Reference to the underlying configuration object from which the profile

       derives its other attributes.

    .. attribute:: key

       String which differentiates this profile from any others which may exist

       within the configuration.

    .. attribute:: dirs

       List of directory paths which should be watched by the file monitor on

       behalf of this profile.

    .. attribute:: watch_locks

       Whether the file monitor should watch for new files, or disappearance of

       file locks.

       If ``False`` (the default), the file monitor will not use any special

       logic and will simply be on the lookout for any new files to appear;

       when they do, the configured action(s) will be invoked.

       If this setting is ``True``, then the simple appearance of a new file

       will not in itself cause any action(s) being taken.  Instead, the file

       monitor will only watch for the *disappearance* of file "locks",

       i.e. any file or directory whose name ends in ``".lock"``.  When the

       disappearance of such a lock is noticed, the configured action(s) will

       be invoked.

    .. attribute:: process_existing

       Whether any pre-existing files present in the watched folder(s) should

       be automatically added to the processing queue.

       If this is ``True`` (the default), then when the file monitor first

       starts, it will examine each watched folder to see if it already

       contains files.  If it does, then each file will be added to the

       processing queue exactly as if the monitor had just noticed the file

       appear.  The files will be added in order of creation timestam, in an

       effort to be consistent with the real-time behavior.

       If this setting is ``False``, any files which happen to exist when the

       file monitor first starts will be ignored.

    .. attribute:: stop_on_error

       Whether the file monitor should halt processing for all future files, if

       a single error is encountered.

       If this is ``True``, then if a single file action raises an exception,

       all subsequent files to appear will be effectively ignored.  No further

       actions will be taken for any of them.

       If this is ``False`` (the default), then a single file error will only

       cause processing to halt for that particular file.  Subsequent files

       which appear will still be processed as normal.

    .. attribute:: actions

       List of :class:`ProfileAction` instances representing the actions to be

       invoked when new files are discovered.

    """

    def __init__(self, config, key):

        self.config = config

        self.key = key

        self.dirs = self.normalize_dirs(self._config_list(u'dirs'))

        self.watch_locks = self._config_boolean(u'watch_locks', False)

        self.process_existing = self._config_boolean(u'process_existing', True)

        self.stop_on_error = self._config_boolean(u'stop_on_error', False)

        self.actions = []

        for action in self._config_list(u'actions'):

            self.actions.append(self._config_action(action))

    def _config_action(self, name):

        function = self._config_string(u'action.{0}.func'.format(name))

        class_ = self._config_string(u'action.{0}.class'.format(name))

        if function and class_:

            raise ConfigurationError(

                u"File monitor profile '{0}' has both function *and* class defined for "

                u"action '{1}' (must have one or the other).".format(self.key, name))

        if not function and not class_:

            raise ConfigurationError(

                u"File monitor profile '{0}' has neither function *nor* class defined for "

                u"action '{1}' (must have one or the other).".format(self.key, name))

        action = ProfileAction()

        if function:

            action.spec = function

            action.action = load_object(action.spec)

        else:

            action.spec = class_

            action.action = load_object(action.spec)(self.config)

        action.args = self._config_list(u'action.{0}.args'.format(name))

        action.kwargs = {}

        pattern = re.compile(ur'^{0}\.action\.{1}\.kwarg\.(?P<keyword>\w+)$'.format(self.key, name), re.IGNORECASE)

        for option in self.config.options(u'rattail.filemon'):

            match = pattern.match(option)

            if match:

                action.kwargs[match.group(u'keyword')] = self.config.get(u'rattail.filemon', option)

        action.retry_attempts = self._config_int(u'action.{0}.retry_attempts'.format(name), minimum=1)

        action.retry_delay = self._config_int(u'action.{0}.retry_delay'.format(name), minimum=0)

        return action

    def _config_boolean(self, option, default=None):

        return self.config.getboolean(

            u'rattail.filemon', u'{0}.{1}'.format(self.key, option), default=default)

    def _config_int(self, option, minimum=0):

        option = u'{0}.{1}'.format(self.key, option)

        if self.config.has_option(u'rattail.filemon', option):

            value = self.config.getint(u'rattail.filemon', option)

            if value < minimum:

                log.warning(u"config value {0} is too small; falling back to minimum "

                            u"of {1} for option: {2}".format(value, minimum, option))

                value = minimum

        else:

            value = minimum

        return value

    def _config_list(self, option):

        return parse_list(self._config_string(option))

    def _config_string(self, option):

        return self.config.get(u'rattail.filemon', u'{0}.{1}'.format(self.key, option))

    def normalize_dirs(self, dirs):

        """

        Normalize a list of directory paths.  Converts all to absolute paths,

        and prunes those which do not exist or do not refer to directories.

        """

        normalized = []

        for path in dirs:

            path = os.path.abspath(path)

            if not os.path.exists(path):

                log.warning(u"pruning folder which does not exist for profile "

                            u"{0}: {1}".format(repr(self.key), repr(path)))

                continue

            if not os.path.isdir(path):

                log.warning(u"pruning folder which is not a directory for profile "

                            u"{0}: {1}".format(repr(self.key), repr(path)))

                continue

            normalized.append(path)

        return normalized

def load_profiles(config):

    """

    Load all active file monitor profiles defined within configuration.

    """

    # Make sure we have a top-level directive.

    keys = config.get(u'rattail.filemon', u'monitor')

    if not keys:

        # No?  What about with the old syntax?

        keys = config.get(u'rattail.filemon', u'monitored')

        if keys:

            return load_legacy_profiles(config)

        # Still no?  That's no good..

        raise ConfigurationError(

            u"The file monitor configuration does not specify any profiles "

            u"to be monitored.  Please defined the 'monitor' option within "

            u"the [rattail.filemon] section of your config file.")

    monitored = {}

    for key in parse_list(keys):

        profile = Profile(config, key)

        if not profile.dirs:

            log.warning(u"profile '{0}' has no valid directories to watch".format(key))

            continue

        if not profile.actions:

            log.warning(u"profile '{0}' has no valid actions to invoke".format(key))

            continue

        monitored[key] = profile

    return monitored

class LegacyProfile(Profile):

    """

    This is a file monitor profile corresponding to the first generation of

    configuration syntax.  E.g. it only supports function-based actions, and

    does not support keyword arguments.

    .. note::

       This profile type will be deprecated at some point in the future.  All

       new code should assume use :class:`Profile` instead.

    """

    def __init__(self, config, key):

        self.config = config

        self.key = key

        dirs = self._config_string(u'dirs')

        if dirs:

            self.dirs = eval(dirs)

        else:

            self.dirs = []

        self.dirs = self.normalize_dirs(self.dirs)

        actions = self._config_string(u'actions')

        if actions:

            actions = eval(actions)

        else:

            actions = []

        self.actions = []

        for action in actions:

            if isinstance(action, tuple):

                spec = action[0]

                args = list(action[1:])

            else:

                spec = action

                args = []

            action = load_object(spec)

            self.actions.append((spec, action, args, {}))

        self.watch_locks = self._config_boolean(u'locks', False)

        self.process_existing = self._config_boolean(u'process_existing', True)

        self.stop_on_error = self._config_boolean(u'stop_on_error', False)

def load_legacy_profiles(config):

    """

    Load all active *legacy* file monitor profiles defined within

    configuration.

    """

    warnings.warn(u"The use of legacy profiles is deprecated, and should be "

                  u"avoided.  Please update your configuration to use the new "

                  u"syntax as soon as possible.", DeprecationWarning)

    monitored = {}

    # Read monitor profile(s) from config.

    keys = config.get(u'rattail.filemon', u'monitored')

    if not keys:

        raise ConfigurationError(

            u"The file monitor configuration does not specify any profiles "

            u"to be monitored.  Please defined the 'monitor' option within "

            u"the [rattail.filemon] section of your config file.")

    keys = keys.split(u',')

    for key in keys:

        key = key.strip()

        log.debug(u"loading profile: {0}".format(key))

        monitored[key] = LegacyProfile(config, key)