worker: Proper SIGTERM handling

Exit immediately and cleanly when receiving SIGTERM during queue wait, but
finish processing a current request to avoid unnecessary re-runs.

This makes it possible to run "killall worker" to restart them all after
configuration or code changes.

1 files changed, 22 insertions, 3 deletions

diff --git a/worker/worker b/worker/worker
index 5bb7093..7817a26 100755
--- a/worker/worker
+++ b/worker/worker
@@ -15,6 +15,7 @@ import logging
 import tempfile
 import shutil
 import re
+import signal
 
 import amqplib.client_0_8 as amqp
 import swiftclient
@@ -23,6 +24,15 @@ my_path = os.path.dirname(__file__)
 args = None
 cfg = None
 swift_creds = {}
+exit_requested = False
+
+
+def term_handler(signum, frame):
+    '''SIGTERM handler'''
+
+    logging.info('Caught SIGTERM, requesting exit')
+    global exit_requested
+    exit_requested = True
 
 
 def parse_args():
@@ -257,6 +267,10 @@ def request(msg):
     logging.info('Acknowledging request %s' % msg.body)
     msg.channel.basic_ack(msg.delivery_tag)
 
+    if exit_requested:
+        logging.info('Exiting due to queued SIGTERM request')
+        sys.exit(0)
+
 
 def amqp_connect(cfg, callback):
     '''Connect to AMQP host using given configuration
@@ -301,6 +315,8 @@ def main():
 
     args = parse_args()
 
+    signal.signal(signal.SIGTERM, term_handler)
+
     # load configuration
     cfg = ConfigParser.ConfigParser(
         {'long_tests': '', 'setup_command': '', 'big_packages': '',
@@ -343,9 +359,12 @@ def main():
         # Retry on e. g. requests.exceptions.ConnectionError connection resets
         # after being idle for some time
         except IOError as e:
-            logging.error('AMQP queue interrupted, reconnecting in 5s: %s' % str(e))
-            time.sleep(5)
-            queue = amqp_connect(cfg, request)
+            if exit_requested:
+                break
+            else:
+                logging.error('AMQP queue interrupted, reconnecting in 5s: %s' % str(e))
+                time.sleep(5)
+                queue = amqp_connect(cfg, request)
 
 
 if __name__ == '__main__':