Files
@ 2c7d6c16e14f
Branch filter:
Location: rattail-project/rattail/rattail/filemon/config.py
2c7d6c16e14f
13.3 KiB
text/x-python
File Monitor overhaul!
* New configuration syntax (old syntax still supported but deprecated).
* Class-based actions.
* Configure keyword arguments to action callables.
* Configure retry for actions.
* Add (some) tests, docs.
* New configuration syntax (old syntax still supported but deprecated).
* Class-based actions.
* Configure keyword arguments to action callables.
* Configure retry for actions.
* Add (some) tests, docs.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 | # -*- coding: utf-8 -*-
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2014 Lance Edgar
#
# This file is part of Rattail.
#
# Rattail is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
# more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Rattail. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
File Monitor Configuration
"""
import os
import re
import shlex
import warnings
import logging
from rattail.util import load_object
from rattail.exceptions import ConfigurationError
log = logging.getLogger(__name__)
class ProfileAction(object):
"""
Simple class to hold configuration for a particular action defined within a
monitor :class:`Profile`. Each instance has the following attributes:
.. attribute:: spec
The original "spec" string used to obtain the action callable.
.. attribute:: action
A reference to the action callable.
.. attribute:: args
A sequence of positional arguments to be passed to the callable (in
addition to the file path) when invoking the action.
.. attribute:: kwargs
A dictionary of keyword arguments to be passed to the callable (in
addition to the positional arguments) when invoking the action.
.. attribute:: retry_attempts
Number of attempts to make when invoking the action. Defaults to ``1``,
meaning the first attempt will be made but no retries will happen.
.. attribute:: retry_delay
Number of seconds to pause between retry attempts, if
:attr:`retry_attempts` is greater than one. Defaults to ``0``.
"""
spec = None
action = None
args = []
kwargs = {}
retry_attempts = 1
retry_delay = 0
class Profile(object):
"""
Simple class to hold configuration for a file monitor "profile". The
profile determines which folders to watch, which actions to take when new
files appear, etc. Each instance of this class has the following
attributes:
.. attribute:: config
Reference to the underlying configuration object from which the profile
derives its other attributes.
.. attribute:: key
String which differentiates this profile from any others which may exist
within the configuration.
.. attribute:: dirs
List of directory paths which should be watched by the file monitor on
behalf of this profile.
.. attribute:: watch_locks
Whether the file monitor should watch for new files, or disappearance of
file locks.
If ``False`` (the default), the file monitor will not use any special
logic and will simply be on the lookout for any new files to appear;
when they do, the configured action(s) will be invoked.
If this setting is ``True``, then the simple appearance of a new file
will not in itself cause any action(s) being taken. Instead, the file
monitor will only watch for the *disappearance* of file "locks",
i.e. any file or directory whose name ends in ``".lock"``. When the
disappearance of such a lock is noticed, the configured action(s) will
be invoked.
.. attribute:: process_existing
Whether any pre-existing files present in the watched folder(s) should
be automatically added to the processing queue.
If this is ``True`` (the default), then when the file monitor first
starts, it will examine each watched folder to see if it already
contains files. If it does, then each file will be added to the
processing queue exactly as if the monitor had just noticed the file
appear. The files will be added in order of creation timestam, in an
effort to be consistent with the real-time behavior.
If this setting is ``False``, any files which happen to exist when the
file monitor first starts will be ignored.
.. attribute:: stop_on_error
Whether the file monitor should halt processing for all future files, if
a single error is encountered.
If this is ``True``, then if a single file action raises an exception,
all subsequent files to appear will be effectively ignored. No further
actions will be taken for any of them.
If this is ``False`` (the default), then a single file error will only
cause processing to halt for that particular file. Subsequent files
which appear will still be processed as normal.
.. attribute:: actions
List of :class:`ProfileAction` instances representing the actions to be
invoked when new files are discovered.
"""
def __init__(self, config, key):
self.config = config
self.key = key
self.dirs = self.normalize_dirs(self._config_list(u'dirs'))
self.watch_locks = self._config_boolean(u'watch_locks', False)
self.process_existing = self._config_boolean(u'process_existing', True)
self.stop_on_error = self._config_boolean(u'stop_on_error', False)
self.actions = []
for action in self._config_list(u'actions'):
self.actions.append(self._config_action(action))
def _config_action(self, name):
function = self._config_string(u'action.{0}.func'.format(name))
class_ = self._config_string(u'action.{0}.class'.format(name))
if function and class_:
raise ConfigurationError(
u"File monitor profile '{0}' has both function *and* class defined for "
u"action '{1}' (must have one or the other).".format(self.key, name))
if not function and not class_:
raise ConfigurationError(
u"File monitor profile '{0}' has neither function *nor* class defined for "
u"action '{1}' (must have one or the other).".format(self.key, name))
action = ProfileAction()
if function:
action.spec = function
action.action = load_object(action.spec)
else:
action.spec = class_
action.action = load_object(action.spec)(self.config)
action.args = self._config_list(u'action.{0}.args'.format(name))
action.kwargs = {}
pattern = re.compile(ur'^{0}\.action\.{1}\.kwarg\.(?P<keyword>\w+)$'.format(self.key, name), re.IGNORECASE)
for option in self.config.options(u'rattail.filemon'):
match = pattern.match(option)
if match:
action.kwargs[match.group(u'keyword')] = self.config.get(u'rattail.filemon', option)
action.retry_attempts = self._config_int(u'action.{0}.retry_attempts'.format(name), minimum=1)
action.retry_delay = self._config_int(u'action.{0}.retry_delay'.format(name), minimum=0)
return action
def _config_boolean(self, option, default=None):
return self.config.getboolean(
u'rattail.filemon', u'{0}.{1}'.format(self.key, option), default=default)
def _config_int(self, option, minimum=0):
option = u'{0}.{1}'.format(self.key, option)
if self.config.has_option(u'rattail.filemon', option):
value = self.config.getint(u'rattail.filemon', option)
if value < minimum:
log.warning(u"config value {0} is too small; falling back to minimum "
u"of {1} for option: {2}".format(value, minimum, option))
value = minimum
else:
value = minimum
return value
def _config_list(self, option):
return parse_list(self._config_string(option))
def _config_string(self, option):
return self.config.get(u'rattail.filemon', u'{0}.{1}'.format(self.key, option))
def normalize_dirs(self, dirs):
"""
Normalize a list of directory paths. Converts all to absolute paths,
and prunes those which do not exist or do not refer to directories.
"""
normalized = []
for path in dirs:
path = os.path.abspath(path)
if not os.path.exists(path):
log.warning(u"pruning folder which does not exist for profile "
u"{0}: {1}".format(repr(self.key), repr(path)))
continue
if not os.path.isdir(path):
log.warning(u"pruning folder which is not a directory for profile "
u"{0}: {1}".format(repr(self.key), repr(path)))
continue
normalized.append(path)
return normalized
def load_profiles(config):
"""
Load all active file monitor profiles defined within configuration.
"""
# Make sure we have a top-level directive.
keys = config.get(u'rattail.filemon', u'monitor')
if not keys:
# No? What about with the old syntax?
keys = config.get(u'rattail.filemon', u'monitored')
if keys:
return load_legacy_profiles(config)
# Still no? That's no good..
raise ConfigurationError(
u"The file monitor configuration does not specify any profiles "
u"to be monitored. Please defined the 'monitor' option within "
u"the [rattail.filemon] section of your config file.")
monitored = {}
for key in parse_list(keys):
profile = Profile(config, key)
if not profile.dirs:
log.warning(u"profile '{0}' has no valid directories to watch".format(key))
continue
if not profile.actions:
log.warning(u"profile '{0}' has no valid actions to invoke".format(key))
continue
monitored[key] = profile
return monitored
class LegacyProfile(Profile):
"""
This is a file monitor profile corresponding to the first generation of
configuration syntax. E.g. it only supports function-based actions, and
does not support keyword arguments.
.. note::
This profile type will be deprecated at some point in the future. All
new code should assume use :class:`Profile` instead.
"""
def __init__(self, config, key):
self.config = config
self.key = key
dirs = self._config_string(u'dirs')
if dirs:
self.dirs = eval(dirs)
else:
self.dirs = []
self.dirs = self.normalize_dirs(self.dirs)
actions = self._config_string(u'actions')
if actions:
actions = eval(actions)
else:
actions = []
self.actions = []
for action in actions:
if isinstance(action, tuple):
spec = action[0]
args = list(action[1:])
else:
spec = action
args = []
action = load_object(spec)
self.actions.append((spec, action, args, {}))
self.watch_locks = self._config_boolean(u'locks', False)
self.process_existing = self._config_boolean(u'process_existing', True)
self.stop_on_error = self._config_boolean(u'stop_on_error', False)
def load_legacy_profiles(config):
"""
Load all active *legacy* file monitor profiles defined within
configuration.
"""
warnings.warn(u"The use of legacy profiles is deprecated, and should be "
u"avoided. Please update your configuration to use the new "
u"syntax as soon as possible.", DeprecationWarning)
monitored = {}
# Read monitor profile(s) from config.
keys = config.get(u'rattail.filemon', u'monitored')
if not keys:
raise ConfigurationError(
u"The file monitor configuration does not specify any profiles "
u"to be monitored. Please defined the 'monitor' option within "
u"the [rattail.filemon] section of your config file.")
keys = keys.split(u',')
for key in keys:
key = key.strip()
log.debug(u"loading profile: {0}".format(key))
monitored[key] = LegacyProfile(config, key)
for key in monitored.keys():
profile = monitored[key]
# Prune any profiles with no valid folders to monitor.
if not profile.dirs:
log.warning(u"profile has no folders to monitor, "
u"and will be pruned: {0}".format(key))
del monitored[key]
# Prune any profiles with no valid actions to perform.
elif not profile.actions:
log.warning(u"profile has no actions to perform, "
u"and will be pruned: {0}".format(key))
del monitored[key]
return monitored
def parse_list(value):
"""
Parse a configuration value, splitting by whitespace and/or commas and
taking quoting into account, etc., yielding a list of strings.
"""
if value is None:
return []
parser = shlex.shlex(value)
parser.whitespace += u','
parser.whitespace_split = True
values = list(parser)
for i, value in enumerate(values):
if value.startswith(u'"') and value.endswith(u'"'):
values[i] = value[1:-1]
return values
|