1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
#!/usr/bin/python3
# clean up broken/orphaned instances
import os
import time
import logging
import subprocess
import novaclient.client
# number of seconds at which an instance counts as "implausibly old"
# long_tests has a timeout of 40.000s ~ 11.1 h
MAX_AGE = 12 * 3600
if os.environ.get('DEBUG'):
logging.basicConfig(level=logging.DEBUG)
nova = novaclient.client.Client(
'2', os.environ['NOVA_USERNAME'], os.environ['NOVA_PASSWORD'],
os.environ['NOVA_PROJECT_ID'], auth_url=os.environ['OS_AUTH_URL'],
region_name=os.environ['NOVA_REGION'])
for instance in nova.servers.list():
age = time.mktime(time.gmtime()) - time.mktime(time.strptime(instance.created, '%Y-%m-%dT%H:%M:%SZ'))
logging.debug('%s: status %s, age %is, networks %s' % (instance.name, instance.status, age, instance.networks))
# check state
if instance.status == 'ERROR':
logging.warning('instance %s is in error state (message: %s), deleting', instance.name, str(instance.fault))
instance.delete()
continue
if not instance.name.startswith('adt-'):
continue
# check age
if age > MAX_AGE:
logging.warning('instance %s is %.1f hours old, deleting', instance.name, (float(age)/3600))
instance.delete()
# check matching adt-run process for instance name
try:
if subprocess.call(['pgrep', '-f', 'python.*autopkgtest.* --name %s' % instance.name],
stdout=subprocess.PIPE) != 0:
logging.warning('instance %s has no associated autopkgtest (auth_url: %s)', instance.name, nova.client.auth_url)
instance.delete()
except IndexError:
logging.warning('instance %s has invalid name' % instance.name)
|
