path: root/worker/worker

#!/usr/bin/python3
# autopkgtest cloud worker
# Author: Martin Pitt <martin.pitt@ubuntu.com>
#
# Requirements: python3-amqplib python3-swiftclient
# Requirements for running autopkgtest from git: python3-debian libdpkg-perl

import os
import sys
import time
import argparse
import configparser
import subprocess
import logging
import tempfile
import shutil
import signal
import json
import urllib.request
import re
import hashlib
import random

import amqplib.client_0_8 as amqp
import swiftclient

from urllib.error import HTTPError

my_path = os.path.dirname(__file__)
root_path = os.path.dirname(os.path.abspath(my_path))
args = None
cfg = None
swift_creds = {}
exit_requested = None
running_test = False
status_exchange_name = 'teststatus.fanout'
amqp_con = None

# In the case of a tmpfail, look for these strings in the log and if they're
# found, consider this a real failure instead. This is useful if the test
# breaks the testbed so much that autopkgtest can't report a failure properly -
# in this case the nova script will output the console log.

# Note that you get *three* tries, and you need to fail in this way every time
# for the failure to be converted from a tmpfail to a real one.
FAIL_STRINGS = ['Kernel panic - not syncing:',
                'Freezing execution.',
                'Out of memory: Kill process',
                'error: you need to load the kernel first.']

# Some packages can provoke specific breakage. For most packages, this would be
# a sign of infrastructure trouble, but for these we should play it safe and
# consider these to be regressions. If they *are* infrastructure problems,
# we'll have to retry them.
FAIL_PKG_STRINGS = {'systemd': ['timed out waiting for testbed to reboot',
                                'Timed out on waiting for ssh connection'],
                    'cluster-glue': ['timed out waiting for testbed to reboot'],
                    'lxc': ['Error starting container']}


def term_handler(signum, frame):
    '''SIGTERM handler, for clean exit after current test'''

    logging.info('Caught SIGTERM, requesting exit')
    global exit_requested
    exit_requested = 0
    if not running_test:
        amqp_con.close()


def hup_handler(signum, frame):
    '''SIGHUP handler, for restarting after current test'''

    logging.info('Caught SIGHUP, requesting restart')
    global exit_requested
    exit_requested = 10
    if not running_test:
        amqp_con.close()


def parse_args():
    '''Parse command line and return argparse.args object'''

    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config',
                        default=os.path.join(my_path, 'worker.conf'),
                        help='configuration file (default: %(default)s)')
    parser.add_argument('-d', '--debug', action='store_true', default=False,
                        help='enable debug logging')
    parser.add_argument('-v', '--variable', metavar='KEY=VALUE',
                        action='append', default=[],
                        help='define additional variable for given config file')
    return parser.parse_args()


def process_output_dir(dir, pkgname, code):
    '''Post-process output directory'''

    files = set(os.listdir(dir))

    # LP: #1641888
    # In failure cases where we don't know the version, write 'unknown' out as
    # the version, so that frontends (e.g. autopkgtest-web, or britney) can
    # display the result.
    if code in (12, 20) and 'testpkg-version' not in files:
        logging.warning('Code %d returned and no testpkg-version - returning "unknown" for %s' % (code, pkgname))
        with open(os.path.join(dir, 'testpkg-version'), 'w') as testpkg_version:
            testpkg_version.write('%s unknown' % pkgname)
        files.add('testpkg-version')

    # these are small and we need only these for gating and indexing
    resultfiles = ['exitcode']
    # these might not be present in infrastructure failure cases
    for f in ['testbed-packages', 'testpkg-version', 'duration', 'testinfo.json']:
        if f in files:
            resultfiles.append(f)
    subprocess.check_call(['tar', 'cf', 'result.tar'] + resultfiles, cwd=dir)
    # keep testbed-packages in artifacts.tar.gz too, as that is practical next
    # to the <test>-packages files
    for f in resultfiles:
        if f == 'testbed-packages':
            continue
        files.discard(f)
        os.unlink(os.path.join(dir, f))

    # compress main log file, for direct access
    subprocess.check_call(['gzip', '-9', os.path.join(dir, 'log')])
    files.discard('log')

    if files:
        # tar up all other artifacts
        subprocess.check_call(['tar', '-czf', 'artifacts.tar.gz'] + list(files), cwd=dir)
        for f in files:
            path = os.path.join(dir, f)
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.unlink(path)


def subst(s, autopkgtest_checkout, big_package, release, architecture, pkgname):
    subst = {
        'CHECKOUTDIR': autopkgtest_checkout,
        'AUTOPKGTEST_CLOUD_DIR': root_path,
        'RELEASE': release,
        'ARCHITECTURE': architecture,
        'PACKAGENAME': pkgname,
        'PACKAGESIZE': cfg.get('virt',
                               big_package and 'package_size_big' or 'package_size_default'),
        'TIMESTAMP': time.strftime('%Y%m%d-%H%M%S'),
        'EXT': '-uefi' if release in ('xenial','trusty') and architecture == 'arm64' else ''
    }
    for i in args.variable:
        k, v = i.split('=', 1)
        subst[k] = v

    for k, v in subst.items():
        s = s.replace('$' + k, v)
    return s


def send_status_info(queue, release, architecture, pkgname, params, out_dir, running, duration):
    '''Send status and logtail to status queue'''

    if not queue:
        return

    # print('status_info:', release, architecture, pkgname, out_dir, running)
    try:
        with open(os.path.join(out_dir, 'log'), 'rb') as f:
            try:
                f.seek(-2000, os.SEEK_END)
                # throw away the first line as we almost surely cut that out in
                # the middle
                f.readline()
            except IOError:
                # file is smaller than 2000 bytes? okay
                pass
            logtail = f.read().decode('UTF-8', errors='replace')
    except (IOError, OSError) as e:
        logtail = 'Cannot read log file: %s' % e

    msg = json.dumps({'release': release,
                      'architecture': architecture,
                      'package': pkgname,
                      'running': running,
                      'params': params,
                      'duration': duration,
                      'logtail': logtail})
    queue.basic_publish(amqp.Message(msg), status_exchange_name, '')


def call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time):
    '''Call autopkgtest and regularly send status/logtail to status_exchange_name

    Return exit code.
    '''
    # set up status AMQP exchange
    global amqp_con
    status_amqp = amqp_con.channel()
    status_amqp.access_request('/data', active=True, read=False, write=True)
    status_amqp.exchange_declare(status_exchange_name, 'fanout', durable=False, auto_delete=True)

    null_fd = open('/dev/null', 'w')
    autopkgtest = subprocess.Popen(argv, stdout=null_fd, stderr=subprocess.STDOUT)
    # FIXME: Use autopkgtest.wait(timeout=10) once moving to Python 3
    # only send status update every 10s, but check if program has finished every 1s
    status_update_counter = 0
    while autopkgtest.poll() is None:
        time.sleep(1)
        status_update_counter = (status_update_counter + 1) % 10
        if status_update_counter == 0:
            send_status_info(status_amqp, release, architecture, pkgname,
                             params, out_dir, True, int(time.time() - start_time))

    ret = autopkgtest.wait()
    send_status_info(status_amqp, release, architecture, pkgname, params,
                     out_dir, False, int(time.time() - start_time))

    return ret


def request(msg):
    '''Callback for AMQP queue request'''

    blacklisted = False

    # FIXME: make this more elegant
    fields = msg.delivery_info['routing_key'].split('-')
    if len(fields) == 4:
        release, architecture = fields[2:4]
    elif len(fields) == 3:
        release, architecture = fields[1:3]
    else:
        raise NotImplementedError('cannot parse queue name %s' % msg.delivery_info['routing_key'])

    body = msg.body
    if isinstance(body, bytes):
        try:
            body = msg.body.decode('UTF-8')
        except UnicodeDecodeError as e:
            logging.error('Bad encoding in request "%s": %s', msg.body, e)
            return

    # request is either a single string pkgname or "pkg_name json_params"
    try:
        req = body.split(None, 1)
        pkgname = req[0]
        if len(req) > 1:
            params = json.loads(req[1])
        else:
            params = {}
    except (ValueError, IndexError):
        logging.error('Received invalid request format "%s"', body)
        return

    if not re.match('[a-zA-Z0-9.+-]+$', pkgname):
        logging.error('Request contains invalid package name, dropping: "%s"', body)
        msg.channel.basic_ack(msg.delivery_tag)
        return

    logging.info('Received request for package %s on %s/%s; params: %s',
                 pkgname, release, architecture, params)

    current_region = os.environ.get('REGION')
    if current_region:
        if ('%s/%s/%s/%s' % (release, architecture, pkgname, current_region) in
                cfg.get('autopkgtest', 'blacklist').split()) or \
            ('all/%s/%s/%s' % (architecture, pkgname, current_region) in
                cfg.get('autopkgtest', 'blacklist').split()):
            logging.warning('Blacklisted on this region (%s), ignoring and sleeping for 5 minutes' % current_region)
            msg.channel.basic_reject(msg.delivery_tag, requeue=True)
            time.sleep(300)
            return

    # build autopkgtest command line
    work_dir = tempfile.mkdtemp(prefix='autopkgtest-work.')
    out_dir = os.path.join(work_dir, 'out')

    if '%s/%s/%s' % (release, architecture, pkgname) in cfg.get('autopkgtest', 'blacklist').split():
        logging.warning('Blacklisted, ignoring')
        blacklisted = True

        # these will be written later on
        code = 99
        duration = 0

        os.makedirs(out_dir)

        # now let's fake up a log file
        with open(os.path.join(out_dir, 'log'), 'w') as log:
            log.write('This package is blacklisted. To get the entry removed, contact a member of the release team.')

        # a json file containing the env
        if 'triggers' in params:
            with open(os.path.join(out_dir, 'testinfo.json'), 'w') as testinfo:
                d = {'custom_environment':
                     ['ADT_TEST_TRIGGERS=%s' % ' '.join(params['triggers'])]}
                json.dump(d, testinfo, indent=True)

        # and the testpackage version (pkgname blacklisted)
        with open(os.path.join(out_dir, 'testpkg-version'), 'w') as testpkg_version:
            testpkg_version.write('%s blacklisted' % pkgname)

    container = 'autopkgtest-' + release
    big_pkg = pkgname in cfg.get('autopkgtest', 'big_packages').strip().split()

    autopkgtest_checkout = cfg.get('autopkgtest', 'checkout_dir').strip()
    if autopkgtest_checkout:
        argv = [os.path.join(autopkgtest_checkout, 'runner', 'autopkgtest')]
    else:
        argv = ['autopkgtest']
    argv += ['--output-dir', out_dir, '--timeout-copy=6000']

    c = cfg.get('autopkgtest', 'extra_args')
    if c:
        argv += c.strip().split()

    c = cfg.get('autopkgtest', 'setup_command').strip()
    if c:
        c = subst(c, autopkgtest_checkout, big_pkg, release, architecture, pkgname)
        argv += ['--setup-commands', c]
    c = cfg.get('autopkgtest', 'setup_command2').strip()
    if c:
        c = subst(c, autopkgtest_checkout, big_pkg, release, architecture, pkgname)
        argv += ['--setup-commands', c]

    if 'ppas' in params and params['ppas']:
        for ppa in params['ppas']:
            try:
                (ppauser, ppaname) = ppa.split('/')
            except ValueError:
                logging.error('Invalid PPA specification, must be lpuser/ppa_name')
                msg.channel.basic_ack(msg.delivery_tag)
                return
            for retry in range(5):
                try:
                    try:
                        f = urllib.request.urlopen('https://api.launchpad.net/1.0/~%s/+archive/ubuntu/%s' % (ppauser, ppaname))
                        contents = f.read().decode('UTF-8')
                        f.close()
                        fingerprint = json.loads(contents)['signing_key_fingerprint']
                        logging.debug('PPA user %s, name %s has GPG fingerprint %s' % (ppauser, ppaname, fingerprint))
                    except HTTPError as e:
                        # It's quite common to get 503s from LP; retry a few times.
                        if e.code != 503:
                            raise
                        logging.warning('Got error 503 from launchpad API')
                        time.sleep(10)
                    else:
                        break
                except (IOError, ValueError, KeyError) as e:
                    logging.error('Cannot get PPA information: "%s". Consuming the request - it will be left dangling; retry once the problem is resolved.' % e)
                    msg.channel.basic_ack(msg.delivery_tag)
                    return
            else:
                logging.error('Cannot contact Launchpad to get PPA information. Consuming the request - it will be left dangling; retry once the problem is resolved.')
                msg.channel.basic_ack(msg.delivery_tag)
                return
            # add GPG key
            argv += ['--setup-commands', 'apt-key adv --keyserver keyserver.ubuntu.com --recv-key ' + fingerprint]
            # add apt source
            argv += ['--setup-commands', 'REL=$(sed -rn "/^(deb|deb-src) .*(ubuntu.com|ftpmaster)/ { s/^[^ ]+ +(\[.*\] *)?[^ ]* +([^ -]+) +.*$/\\2/p; q }" /etc/apt/sources.list); '
                     'echo "deb http://ppa.launchpad.net/%(u)s/%(p)s/ubuntu $REL main" > /etc/apt/sources.list.d/autopkgtest-%(u)s-%(p)s.list; '
                     'echo "deb-src http://ppa.launchpad.net/%(u)s/%(p)s/ubuntu $REL main" >> /etc/apt/sources.list.d/autopkgtest-%(u)s-%(p)s.list;' %
                     {'u': ppauser, 'p': ppaname}]

        # put results into separate container, named by the last PPA
        container += '-%s-%s' % (ppauser, ppaname)

    # only install the triggering package from -proposed, rest from -release
    # this provides better isolation between -proposed packages; but only do
    # that for Ubuntu itself, not for things from git, PPAs, etc.
    # also skip that for the kernel as the linux vs. linux-meta split always
    # screws up the apt pinning
    if cfg.get('virt', 'args') != 'null':
        if 'test-git' not in params and 'test-bzr' not in params and ('ppas' not in params or 'all-proposed' in params):
            pocket_arg = '--apt-pocket=proposed'
            if 'all-proposed' not in params and not pkgname.startswith('linux'):
                trigs = ['src:' + t.split('/', 1)[0] for t in params.get('triggers', [])]
                if trigs:
                    pocket_arg += '=' + ','.join(trigs)
            argv.append(pocket_arg)
        argv.append('--apt-upgrade')

    # determine which test to run
    if 'test-git' in params:
        testargs = ['--no-built-binaries', params['test-git']]
    elif 'build-git' in params:
        testargs = [params['build-git']]
    elif 'test-bzr' in params:
        checkout_dir = os.path.join(work_dir, 'checkout')
        subprocess.check_call(['bzr', 'checkout', '--lightweight', params['test-bzr'], checkout_dir])
        testargs = ['--no-built-binaries', checkout_dir]
    else:
        testargs = [pkgname]

    argv += testargs
    if args.debug:
        argv.append('--debug')
    if pkgname in cfg.get('autopkgtest', 'long_tests').strip().split():
        argv.append('--timeout-test=40000')
    elif big_pkg:
        argv.append('--timeout-test=20000')

    for e in params.get('env', []):
        argv.append('--env=%s' % e)

    if 'triggers' in params:
        argv.append('--env=ADT_TEST_TRIGGERS=%s' % ' '.join(params['triggers']))

        # want to run against a non-default kernel?
        for t in params['triggers']:
            if t.startswith('linux-meta'):
                flavor = t.split('/')[0].replace('linux-meta', '')
                # HWE kernels have their official release name in the binary package names.
                if flavor.startswith('-hwe'):
                    if release == 'xenial':
                        flavor = flavor.replace('-hwe', '-hwe-16.04', 1)
                if flavor == '-ti-omap4':
                    # yay consistency
                    argv += ['--setup-commands', 'apt-get install -y linux-omap4']
                elif release == 'precise' and architecture == 'armhf' and flavor == '':
                    # no linux-image-generic in precise/armhf yet
                    argv += ['--setup-commands', 'apt-get install -y linux-image-omap linux-headers-omap']
                else:
                    argv += ['--setup-commands', 'apt-get install -y linux-image%(f)s linux-headers%(f)s || apt-get install -y linux-image-generic%(f)s linux-headers-generic%(f)s' %
                             {'f': flavor}]
                break

    if 'testname' in params:
        argv.append('--testname=%s' % params['testname'])

    argv.append('--')
    argv += subst(cfg.get('virt', 'args'), autopkgtest_checkout, big_pkg,
                  release, architecture, pkgname).split()

    # run autopkgtest; retry up to three times on tmpfail issues
    if not blacklisted:
        global running_test
        running_test = True
        start_time = time.time()
        num_failures = 0
        for retry in range(3):
            retry_start_time = time.time()
            logging.info('Running %s', ' '.join(argv))
            code = call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time)
            if code == 16 or code < 0:
                if exit_requested is not None:
                    logging.warning('Testbed failure and exit %i requested', exit_requested)
                    sys.exit(exit_requested)
                try:
                    with open(os.path.join(out_dir, 'log')) as f:
                        contents = f.read()
                        # Get the package-specific string for triggers too, since they might have broken the run
                        trigs = [t.split('/', 1)[0] for t in params.get('triggers', [])]
                        fail_trigs = [j for i in [FAIL_PKG_STRINGS.get(trig, []) for trig in trigs] for j in i]

                        # Or if all-proposed, just give up and accept everything
                        fail_all_proposed = [j for i in FAIL_PKG_STRINGS.values() for j in i]

                        allowed_fail_strings = set(FAIL_STRINGS + \
                                                   FAIL_PKG_STRINGS.get(pkgname, []) + \
                                                   fail_trigs + \
                                                   (fail_all_proposed if 'all-proposed' in params else []))

                        fails = [s for s in allowed_fail_strings if s in contents]
                        if fails:
                            num_failures += 1
                            logging.warning('Saw %s in log, which is a sign of a real (not tmp) failure - seen %d so far',
                                            ' and '.join(fails), num_failures)
                except IOError as e:
                    logging.error('Could not read log file: %s' % str(e))
                logging.warning('Testbed failure, retrying in 5 minutes')
                # we need to empty the --output-dir for the next run, otherwise
                # autopkgtest complains
                if retry < 2:
                    shutil.rmtree(out_dir)
                    os.mkdir(out_dir)
                    running_test = False
                    time.sleep(300)
                    running_test = True
            else:
                break
        else:
            if num_failures >= 3:
                logging.warning('Three fails in a row - considering this a failure rather than tmpfail')
                code = 4
            else:
                logging.error('Three tmpfails in a row, aborting worker. Log follows:')
                try:
                    with open(os.path.join(out_dir, 'log')) as f:
                        logging.error(f.read())
                except IOError as e:
                    logging.error('Could not read log file: %s' % str(e))
                sys.exit(99)

        duration = int(time.time() - retry_start_time)

    logging.info('autopkgtest exited with code %i', code)
    if code == 1:
        logging.error('autopkgtest exited with unexpected error code 1')
        sys.exit(1)
    with open(os.path.join(out_dir, 'exitcode'), 'w') as f:
        f.write('%i\n' % code)
    with open(os.path.join(out_dir, 'duration'), 'w') as f:
        f.write('%u\n' % duration)

    process_output_dir(out_dir, pkgname, code)

    # If two tests for the same package with different triggers finish at the
    # same second, we get collisions with just the timestamp; disambiguate with
    # the hashed params. We append a '@' which is a nice delimiter for querying
    # runs in swift.
    run_id = '%s_%s@' % (
        time.strftime('%Y%m%d_%H%M%S', time.gmtime()),
        hashlib.sha1(body.encode('UTF-8')).hexdigest()[:5])
    if pkgname.startswith('lib'):
        prefix = pkgname[:4]
    else:
        prefix = pkgname[0]
    swift_dir = os.path.join(release, architecture, prefix, pkgname, run_id)

    # publish results into swift
    logging.info('Putting results into swift %s %s', container, swift_dir)

    # create it if it does not exist yet
    swift_con = swiftclient.Connection(**swift_creds)
    try:
        swift_con.get_container(container, limit=1)
    except swiftclient.exceptions.ClientException:
        logging.info('container %s does not exist, creating it', container)
        # make it publicly readable
        swift_con.put_container(container, headers={'X-Container-Read': '.rlistings,.r:*'})
        # wait until it exists
        timeout = 50
        while timeout > 0:
            try:
                swift_con.get_container(container, limit=1)
                logging.debug('newly created container %s exists now', container)
                break
            except swiftclient.exceptions.ClientException:
                logging.debug('newly created container %s does not exist yet, continuing poll', container)
                time.sleep(1)
                timeout -= 1
        else:
            logging.error('timed out waiting for newly created container %s', container)
            sys.exit(1)

    for f in os.listdir(out_dir):
        path = os.path.join(out_dir, f)
        with open(path, 'rb') as fd:
            if path.endswith('log.gz'):
                content_type = 'text/plain; charset=UTF-8'
                headers = {'Content-Encoding': 'gzip'}
            else:
                content_type = None
                headers = None
            # swift_con.put_object() is missing the name kwarg
            swiftclient.put_object(swift_con.url, token=swift_con.token,
                                   container=container,
                                   name=os.path.join(swift_dir, f),
                                   contents=fd,
                                   content_type=content_type,
                                   headers=headers,
                                   content_length=os.path.getsize(path))

    swift_con.close()
    shutil.rmtree(work_dir)

    logging.info('Acknowledging request %s' % body)
    msg.channel.basic_ack(msg.delivery_tag)
    running_test = False


def amqp_connect(cfg, callback):
    '''Connect to AMQP host using given configuration

    Connect "callback" to queues for all configured releases and
    architectures.

    Return queue object.
    '''
    global amqp_con
    logging.info('Connecting to AMQP server %s', cfg.get('amqp', 'host'))
    if cfg.has_option('amqp', 'user') and cfg.get('amqp', 'user'):
        amqp_con = amqp.Connection(cfg.get('amqp', 'host'),
                                   userid=cfg.get('amqp', 'user'),
                                   password=cfg.get('amqp', 'password'))
    else:
        amqp_con = amqp.Connection(cfg.get('amqp', 'host'))
    queue = amqp_con.channel()
    # avoids greedy grabbing of the entire queue while being too busy
    queue.basic_qos(0, 1, True)

    arch_str = cfg.get('autopkgtest', 'architectures')
    arch_str = subst(arch_str, 'n/a', 'n/a', 'n/a', 'n/a', 'n/a')
    arches = arch_str.split()
    if not arches:
        my_arch = subprocess.check_output(['dpkg', '--print-architecture'],
                                          universal_newlines=True).strip()
        logging.info('No architectures in configuration, defaulting to %s', my_arch)
        arches = [my_arch]

    # avoid preferring the same architecture on all workers
    queues = []

    contexts = ['', 'huge-', 'ppa-']

    # crude way to not allow upstream tests to monopolise resources - only 50%
    # of workers will take them
    if random.randint(1,100) < 50:
        contexts += ['upstream-']

    for release in cfg.get('autopkgtest', 'releases').split():
        for context in contexts:
            for arch in arches:
                queue_name = 'debci-%s%s-%s' % (context, release, arch)
                queues.append(queue_name)

    random.shuffle(queues)

    for queue_name in queues:
        logging.info('Setting up and listening to AMQP queue %s', queue_name)
        queue.queue_declare(queue_name, durable=True, auto_delete=False)
        queue.basic_consume(queue=queue_name, callback=request)

    return queue


def main():
    '''Main program'''

    global cfg, args, swift_creds

    args = parse_args()

    signal.signal(signal.SIGTERM, term_handler)
    signal.signal(signal.SIGHUP, hup_handler)

    # load configuration
    cfg = configparser.ConfigParser(
        {'setup_command': '', 'setup_command2': '',
         'long_tests': '', 'big_packages': '',
         'checkout_dir': '', 'blacklist': '',
         'package_size_default': '', 'package_size_big': '',
         'extra_args': '',
         'region_name': os.environ.get('OS_REGION_NAME', '<undefined swift region>'),
         'username': os.environ.get('OS_USERNAME', '<undefined swift user>'),
         'auth_url': os.environ.get('OS_AUTH_URL', '<undefined swift auth URL>'),
         'password': os.environ.get('OS_PASSWORD', '<undefined swift password>'),
         'tenant': os.environ.get('OS_TENANT_NAME', '<undefined swift tenant name>')},
        allow_no_value=True)
    cfg.read(args.config)

    logging.basicConfig(level=(args.debug and logging.DEBUG or logging.INFO),
                        format='%(asctime)s [%(process)d] %(levelname)s: %(message)s')

    # build swift credentials
    os_options = {}
    os_options['region_name'] = cfg.get('swift', 'region_name')
    if '/v2.0' in cfg.get('swift', 'auth_url'):
        auth_version = '2.0'
    else:
        auth_version = '1'
    swift_creds = {'authurl': cfg.get('swift', 'auth_url'),
                   'user': cfg.get('swift', 'username'),
                   'key': cfg.get('swift', 'password'),
                   'tenant_name': cfg.get('swift', 'tenant'),
                   'os_options': os_options,
                   'auth_version': auth_version}

    # ensure that we can connect to swift
    swiftclient.Connection(**swift_creds).close()

    # connect to AMQP queues
    queue = amqp_connect(cfg, request)

    # process queues forever
    try:
        while exit_requested is None:
            logging.info('Waiting for and processing AMQP requests')
            queue.wait()
    except IOError:
        if exit_requested is None:
            raise


if __name__ == '__main__':
    main()
    if exit_requested:
        logging.info('Exiting with %i due to queued exit request' % exit_requested)
        sys.exit(exit_requested)