Changeset - 067c02e965b9
[Not reviewed]
0 1 5
Lance Edgar - 5 years ago 2019-09-27 15:51:10
lance@edbob.org
Add logic for installing luigi service as part of backup app
6 files changed with 212 insertions and 1 deletions:
0 comments (0 inline, 0 general) First comment
rattail_fabric2/backup.py
Show inline comments
 
@@ -27,7 +27,7 @@ Fabric library for Backup app
 
import os
 
import datetime
 

	
 
from rattail_fabric2 import borg, python, exists, make_deploy, mkdir, UNSPECIFIED
 
from rattail_fabric2 import apt, borg, python, exists, make_deploy, mkdir, UNSPECIFIED
 

	
 

	
 
deploy_generic = make_deploy(__file__)
 
@@ -55,6 +55,9 @@ def deploy_backup_everything(c, **context):
 
def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
 
                      install_borg=False,
 
                      link_borg_to_bin=True,
 
                      install_luigi=False,
 
                      luigi_history_db=None,
 
                      luigi_listen_address='0.0.0.0',
 
                      install_rattail=True,
 
                      config=None,
 
                      rattail_backup_script=None,
 
@@ -79,6 +82,9 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
 
    if install_borg:
 
        borg.install_dependencies(c)
 

	
 
    if install_luigi:
 
        c.sudo('supervisorctl stop backup:')
 

	
 
    # virtualenv
 
    if mkvirtualenv:
 
        python.mkvirtualenv(c, envname, python='/usr/bin/python3', runas_user=user)
 
@@ -128,6 +134,50 @@ def deploy_backup_app(c, deploy, envname, mkvirtualenv=True, user='rattail',
 
        if link_borg_to_bin:
 
            c.sudo("ln -sf {}/bin/borg /usr/local/bin/borg".format(envpath))
 

	
 
    # luigi
 
    if install_luigi:
 
        packages = ['luigi']
 
        if luigi_history_db:
 
            packages.append('SQLAlchemy')
 
            if luigi_history_db.startswith('postgresql://'):
 
                packages.append('psycopg2')
 
        c.sudo("bash -l -c 'workon {}; pip install {}'".format(envname, ' '.join(packages)), user=user)
 

	
 
        # basic config
 
        mkdir(c, ['{}/app/luigitasks'.format(envpath),
 
                  '{}/app/luigi'.format(envpath),
 
                  '{}/app/luigi/log'.format(envpath),
 
                  '{}/app/work'.format(envpath),
 
        ], owner=user, use_sudo=True)
 
        c.sudo('touch {}/app/luigitasks/__init__.py'.format(envpath), user=user)
 
        deploy_generic(c, 'backup/luigi.cfg.mako', '{}/app/luigi/luigi.cfg'.format(envpath),
 
                       owner=user, mode='0600', use_sudo=True,
 
                       context={'envpath': envpath, 'history_db': luigi_history_db})
 
        deploy_generic(c, 'backup/luigi-logging.conf.mako', '{}/app/luigi/luigi-logging.conf'.format(envpath),
 
                       owner=user, use_sudo=True, context={'envpath': envpath})
 

	
 
        # app/luigitasks/overnight.py - should define the OvernightBackups wrapper task
 
        path = '{}/luigi-overnight.py'.format(envname)
 
        if deploy.local_exists(path):
 
            deploy(c, path, '{}/app/luigitasks/overnight.py'.format(envpath),
 
                   owner=user, use_sudo=True)
 
        else:
 
            deploy_generic(c, 'backup/luigi-overnight.py', '{}/app/luigitasks/overnight.py'.format(envpath),
 
                           owner=user, use_sudo=True)
 

	
 
        # app/overnight-backups.sh - generic script to invoke OvernightBackups task
 
        deploy_generic(c, 'backup/overnight-backups.sh.mako', '{}/app/overnight-backups.sh'.format(envpath),
 
                       owner=user, mode='0755', use_sudo=True, context={'envpath': envpath})
 

	
 
        # supervisor / luigid
 
        apt.install(c, 'supervisor')
 
        deploy_generic(c, 'backup/supervisor.conf.mako', '/etc/supervisor/conf.d/backup.conf',
 
                       use_sudo=True, context={
 
                           'envpath': envpath, 'user': user,
 
                           'listen_address': luigi_listen_address})
 
        c.sudo('supervisorctl update')
 
        c.sudo('supervisorctl start backup:')
 

	
 
    # backup-everything script
 
    if install_rattail or everything:
 
        everything_context = dict(context)
rattail_fabric2/deploy/backup/luigi-logging.conf.mako
Show inline comments
 
new file 100644
 
## -*- mode: conf; -*-
 

	
 
${'#'}###########################################################
 
#
 
# Luigi logging config
 
#
 
${'#'}###########################################################
 

	
 

	
 
[loggers]
 
keys = root
 

	
 
[handlers]
 
keys = file, console
 

	
 
[formatters]
 
keys = generic, console
 

	
 
[logger_root]
 
handlers = file, console
 
level = DEBUG
 

	
 
[handler_file]
 
class = handlers.RotatingFileHandler
 
args = ('${envpath}/app/luigi/log/luigi.log', 'a', 1000000, 20, 'utf_8')
 
formatter = generic
 

	
 
[handler_console]
 
class = StreamHandler
 
args = (sys.stderr,)
 
formatter = console
 
level = WARNING
 

	
 
[formatter_generic]
 
format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
 
datefmt = %Y-%m-%d %H:%M:%S
 

	
 
[formatter_console]
 
format = %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
rattail_fabric2/deploy/backup/luigi-overnight.py
Show inline comments
 
new file 100644
 
# -*- coding: utf-8; -*-
 
"""
 
Luigi tasks for "overnight backups"
 
"""
 

	
 
import subprocess
 
import logging
 

	
 
import luigi
 

	
 
from rattail.luigi import OvernightTask
 
from rattail.logging import LuigiSummaryFilter
 

	
 

	
 
# log WARNING for Luigi execution summary
 
logging.getLogger('luigi-interface').addFilter(LuigiSummaryFilter())
 

	
 

	
 
class BackupSomething(OvernightTask):
 
    """
 
    Backup the 'something' machine.
 
    """
 
    filename = 'backup-something'
 

	
 
    def run_command(self):
 
        # note, this command would assume a "local" machine backup
 
        # subprocess.check_call([
 
        #     'sudo', '/usr/local/bin/backup-everything',
 
        # ])
 
        print('backed something up!')
 

	
 

	
 
class BackupAnother(OvernightTask):
 
    """
 
    Backup the 'another' machine.
 
    """
 
    filename = 'backup-another'
 

	
 
    # note, you must daisy-chain the tasks together, so each task "requires"
 
    # the previous task.  (there probably should be a better way though?)
 
    # our goal with that is just to make sure they run sequentially.
 
    def requires(self):
 
        return BackupSomething(self.date)
 

	
 
    def run_command(self):
 
        # note, this command would assume a "remote" machine backup
 
        # (also assumes ssh keys/config have already been established)
 
        # subprocess.check_call([
 
        #     'ssh', '-o', 'ServerAliveInterval=120', 'another.example.com',
 
        #     'sudo', '/usr/local/bin/backup-everything',
 
        # ])
 
        print('backed another up!')
 

	
 

	
 
class OvernightBackups(luigi.WrapperTask):
 
    """
 
    Wrapper task for "overnight-backups" automation
 
    """
 
    date = luigi.DateParameter()
 

	
 
    # this is our "wrapper" task which is invoked from `overnight-backups.sh`
 
    # we list each sequential task here for clarity, even though that may be
 
    # redundant due to how we daisy-chain them via requires() above (i.e. we
 
    # might be able to just "require" the last task here? needs testing)
 
    def requires(self):
 
        yield BackupSomething(self.date)
 
        yield BackupAnother(self.date)
rattail_fabric2/deploy/backup/luigi.cfg.mako
Show inline comments
 
new file 100644
 
## -*- mode: conf; -*-
 

	
 
${'#'}###########################################################
 
#
 
# Luigi config
 
#
 
${'#'}###########################################################
 

	
 

	
 
[scheduler]
 
state_path = ${envpath}/app/luigi/state.pickle
 
% if history_db:
 
record_task_history = true
 

	
 
[task_history]
 
db_connection = ${history_db}
 
% endif
rattail_fabric2/deploy/backup/overnight-backups.sh.mako
Show inline comments
 
new file 100755
 
#!/bin/bash
 
${'#'}###############################################################################
 
#
 
# overnight "backups" automation
 
#
 
${'#'}###############################################################################
 

	
 
set -e
 

	
 
DATE=$1
 

	
 
if [ "$1" = "--verbose" ]; then
 
    DATE=$2
 
    VERBOSE='--verbose'
 
else
 
    VERBOSE=
 
fi
 

	
 
if [ "$DATE" = "" ]; then
 
    DATE=`date --date='yesterday' +%Y-%m-%d`
 
fi
 

	
 
LUIGI='${envpath}/bin/luigi --logging-conf-file luigi-logging.conf'
 
export PYTHONPATH=${envpath}/app/
 

	
 
cd ${envpath}/app/luigi
 

	
 
$LUIGI --module luigitasks.overnight OvernightBackups --date $DATE
rattail_fabric2/deploy/backup/supervisor.conf.mako
Show inline comments
 
new file 100644
 
## -*- mode: conf; -*-
 

	
 
[group:backup]
 
programs=luigid
 

	
 
[program:luigid]
 
command=${envpath}/bin/luigid --logdir ${envpath}/app/luigi/log --state-path ${envpath}/app/luigi/state.pickle --address ${listen_address}
 
directory=${envpath}/app/work
 
environment=LUIGI_CONFIG_PATH="${envpath}/app/luigi/luigi.cfg"
 
user=${user}
0 comments (0 inline, 0 general) First comment
You need to be logged in to comment. Login now