test: Enable the KVM platform on integration tests

The KVM platform includes:

  * Ubuntu images from daily stream
  * Image setup via mount-image-callback
  * Generation and injection of SSH key pair (RSA 4096)
  * Launching via QEMU CLI and execution via SSH

13 files changed, 1146 insertions, 4 deletions

diff --git a/tests/cloud_tests/__main__.py b/tests/cloud_tests/__main__.py
index 260ddb3..7ee29ca 100644
--- a/tests/cloud_tests/__main__.py
+++ b/tests/cloud_tests/__main__.py
@@ -4,6 +4,7 @@
 
 import argparse
 import logging
+import os
 import sys
 
 from tests.cloud_tests import args, bddeb, collect, manage, run_funcs, verify
@@ -50,7 +51,7 @@ def main():
             return -1
 
     # run handler
-    LOG.debug('running with args: %s\n', parsed)
+    LOG.debug('running with args: %s', parsed)
     return {
         'bddeb': bddeb.bddeb,
         'collect': collect.collect,
@@ -63,6 +64,8 @@ def main():
 
 
 if __name__ == "__main__":
+    if os.geteuid() == 0:
+        sys.exit('Do not run as root')
     sys.exit(main())
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py
index 369d60d..c6c1877 100644
--- a/tests/cloud_tests/args.py
+++ b/tests/cloud_tests/args.py
@@ -170,9 +170,9 @@ def normalize_collect_args(args):
     @param args: parsed args
     @return_value: updated args, or None if errors occurred
     """
-    # platform should default to all supported
+    # platform should default to lxd
     if len(args.platform) == 0:
-        args.platform = config.ENABLED_PLATFORMS
+        args.platform = ['lxd']
     args.platform = util.sorted_unique(args.platform)
 
     # os name should default to all enabled
diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py
index b44e8bd..4a2422e 100644
--- a/tests/cloud_tests/collect.py
+++ b/tests/cloud_tests/collect.py
@@ -120,6 +120,7 @@ def collect_image(args, platform, os_name):
     os_config = config.load_os_config(
         platform.platform_name, os_name, require_enabled=True,
         feature_overrides=args.feature_override)
+    LOG.debug('os config: %s', os_config)
     component = PlatformComponent(
         partial(images.get_image, platform, os_config))
 
@@ -144,6 +145,8 @@ def collect_platform(args, platform_name):
 
     platform_config = config.load_platform_config(
         platform_name, require_enabled=True)
+    platform_config['data_dir'] = args.data_dir
+    LOG.debug('platform config: %s', platform_config)
     component = PlatformComponent(
         partial(platforms.get_platform, platform_name, platform_config))
 
diff --git a/tests/cloud_tests/config.py b/tests/cloud_tests/config.py
index 4d5dc80..52fc2bd 100644
--- a/tests/cloud_tests/config.py
+++ b/tests/cloud_tests/config.py
@@ -112,6 +112,7 @@ def load_os_config(platform_name, os_name, require_enabled=False,
     feature_conf = main_conf['features']
     feature_groups = conf.get('feature_groups', [])
     overrides = merge_config(get(conf, 'features'), feature_overrides)
+    conf['arch'] = c_util.get_architecture()
     conf['features'] = merge_feature_groups(
         feature_conf, feature_groups, overrides)
 
diff --git a/tests/cloud_tests/images/kvm.py b/tests/cloud_tests/images/kvm.py
new file mode 100644
index 0000000..008acef
--- /dev/null
+++ b/tests/cloud_tests/images/kvm.py
@@ -0,0 +1,75 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""KVM Image Base Class."""
+
+from tests.cloud_tests.images import base
+from tests.cloud_tests.snapshots import kvm as kvm_snapshot
+
+
+class KVMImage(base.Image):
+    """KVM backed image."""
+
+    platform_name = "kvm"
+
+    def __init__(self, platform, config, img_path):
+        """Set up image.
+
+        @param platform: platform object
+        @param config: image configuration
+        """
+        self.modified = False
+        self._instance = None
+        self._img_path = img_path
+
+        super(KVMImage, self).__init__(platform, config)
+
+    @property
+    def instance(self):
+        """Property function."""
+        if not self._instance:
+            self._instance = self.platform.create_image(
+                self.properties, self.config, self.features, self._img_path,
+                image_desc=str(self), use_desc='image-modification')
+        return self._instance
+
+    @property
+    def properties(self):
+        """{} containing: 'arch', 'os', 'version', 'release'."""
+        return {
+            'arch': self.config['arch'],
+            'os': self.config['family'],
+            'release': self.config['release'],
+            'version': self.config['version'],
+        }
+
+    def execute(self, *args, **kwargs):
+        """Execute command in image, modifying image."""
+        return self.instance.execute(*args, **kwargs)
+
+    def push_file(self, local_path, remote_path):
+        """Copy file at 'local_path' to instance at 'remote_path'."""
+        return self.instance.push_file(local_path, remote_path)
+
+    def run_script(self, *args, **kwargs):
+        """Run script in image, modifying image.
+
+        @return_value: script output
+        """
+        return self.instance.run_script(*args, **kwargs)
+
+    def snapshot(self):
+        """Create snapshot of image, block until done."""
+        instance = self.platform.create_image(
+            self.properties, self.config, self.features,
+            self._img_path, image_desc=str(self), use_desc='snapshot')
+
+        return kvm_snapshot.KVMSnapshot(
+            self.platform, self.properties, self.config,
+            self.features, instance)
+
+    def destroy(self):
+        """Clean up data associated with image."""
+        self._img_path = None
+        super(KVMImage, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/instances/kvm.py b/tests/cloud_tests/instances/kvm.py
new file mode 100644
index 0000000..5ef884b
--- /dev/null
+++ b/tests/cloud_tests/instances/kvm.py
@@ -0,0 +1,187 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base KVM instance."""
+import os
+import paramiko
+import shlex
+import socket
+import subprocess
+import time
+
+from cloudinit import util as c_util
+from tests.cloud_tests.instances import base
+
+
+class KVMInstance(base.Instance):
+    """KVM backed instance."""
+
+    platform_name = "kvm"
+
+    def __init__(self, platform, name, properties, config, features,
+                 user_data, meta_data):
+        """Set up instance.
+
+        @param platform: platform object
+        @param name: image path
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        """
+        self.user_data = user_data
+        self.meta_data = meta_data
+        self.ssh_key_file = os.path.join(platform.config['data_dir'],
+                                         platform.config['private_key'])
+        self.ssh_port = None
+        self.pid = None
+        self.pid_file = None
+
+        super(KVMInstance, self).__init__(
+            platform, name, properties, config, features)
+
+    def destroy(self):
+        """Clean up instance."""
+        if self.pid:
+            c_util.subp(['kill', '-9', self.pid])
+            os.remove(self.pid_file)
+        super(KVMInstance, self).destroy()
+
+    def execute(self, command, stdout=None, stderr=None, env={},
+                rcs=None, description=None):
+        """Execute command in instance.
+
+        Assumes functional networking and execution as root with the
+        target filesystem being available at /.
+
+        @param command: the command to execute as root inside the image
+        @param stdout, stderr: file handles to write output and error to
+        @param env: environment variables
+        @param rcs: allowed return codes from command
+        @param description: purpose of command
+        @return_value: tuple containing stdout data, stderr data, exit code
+        """
+        if self.pid:
+            out, err = self.ssh(command)
+        else:
+            out, err = self.mount_image_callback(command)
+
+        return out, err
+
+    def mount_image_callback(self, command,):
+        """Run mount-image-callback."""
+        out, err = c_util.subp(['sudo', 'mount-image-callback',
+                                '--system-mounts', '--system-resolvconf',
+                                self.name, '--', 'chroot', '_MOUNTPOINT_',
+                                '/bin/sh', '-c'] + command)
+
+        return out, err
+
+    def generate_seed(self, tmpdir):
+        """Generate nocloud seed from user-data"""
+        seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name)
+        user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name)
+
+        if os.path.exists(seed_file):
+            os.remove(seed_file)
+        if os.path.exists(user_data_file):
+            os.remove(user_data_file)
+
+        with open(user_data_file, "w") as ud_file:
+            ud_file.write(self.user_data)
+
+        c_util.subp(['cloud-localds', seed_file, user_data_file])
+
+        return seed_file
+
+    def get_free_port(self):
+        """Get a free port assigned by the kernel."""
+        s = socket.socket()
+        s.bind(('', 0))
+        num = s.getsockname()[1]
+        s.close()
+        return num
+
+    def push_file(self, local_path, remote_path, description=''):
+        """Copy file at 'local_path' to instance at 'remote_path'.
+
+        If we have a pid then SSH is up, otherwise, use
+        mount-image-callback.
+
+        @param local_path: path on local instance
+        @param remote_path: path on remote instance
+        """
+        if self.pid:
+            super(KVMInstance, self).push_file()
+        else:
+            self.mount_image_callback(['cat', '>', remote_path,
+                                       '<', local_path])
+
+    def sftp_put(self, path, data):
+        """SFTP put a file."""
+        client = self._ssh_connect()
+        sftp = client.open_sftp()
+
+        with sftp.open(path, 'w') as f:
+            f.write(data)
+
+        client.close()
+
+    def ssh(self, command):
+        """Run a command via SSH."""
+        client = self._ssh_connect()
+        _, out, err = client.exec_command(command)
+        exit = out.channel.recv_exit_status()
+        out = ''.join(out.readlines())
+        err = ''.join(err.readlines())
+        client.close()
+
+        return out, err, exit
+
+    def _ssh_connect(self, hostname='localhost', username='ubuntu',
+                     banner_timeout=120, retry_attempts=30):
+        """Connect via SSH."""
+        private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file)
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        while retry_attempts:
+            try:
+                client.connect(hostname=hostname, username=username,
+                               port=self.ssh_port, pkey=private_key,
+                               banner_timeout=banner_timeout)
+                break
+            except paramiko.SSHException:
+                time.sleep(1)
+                retry_attempts = retry_attempts - 1
+
+        return client
+
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance."""
+        tmpdir = self.platform.config['data_dir']
+        seed = self.generate_seed(tmpdir)
+        self.pid_file = os.path.join(tmpdir, self.name, 'pid')
+        self.ssh_port = self.get_free_port()
+
+        cmd = ('./tools/xkvm --disk %s --disk %s '
+               '--netdev user,hostfwd=tcp::%s-:22 '
+               '-- -pidfile %s -vnc none'
+               % (self.name, seed, self.ssh_port, self.pid_file))
+
+        subprocess.Popen(shlex.split(cmd), close_fds=True)
+        while not os.path.exists(self.pid_file):
+            time.sleep(1)
+
+        with open(self.pid_file, 'r') as pid_f:
+            self.pid = pid_f.readlines()[0].strip()
+
+        if wait:
+            self._wait_for_system(wait_for_cloud_init)
+
+    def write_data(self, remote_path, data):
+        """Write data to instance filesystem.
+
+        @param remote_path: path in instance
+        @param data: data to write, either str or bytes
+        """
+        self.sftp_put(remote_path, data)
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml
index b91834a..7bdc1f5 100644
--- a/tests/cloud_tests/platforms.yaml
+++ b/tests/cloud_tests/platforms.yaml
@@ -8,6 +8,10 @@ default_platform_config:
     create_instance_timeout: 60
 
 platforms:
+    kvm:
+        enabled: true
+        private_key: id_rsa
+        public_key: id_rsa.pub
     lxd:
         enabled: true
         # overrides for image templates
diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py
index 443f6d4..7890dae 100644
--- a/tests/cloud_tests/platforms/__init__.py
+++ b/tests/cloud_tests/platforms/__init__.py
@@ -2,10 +2,12 @@
 
 """Main init."""
 
+from tests.cloud_tests.platforms import kvm
 from tests.cloud_tests.platforms import lxd
 
 PLATFORMS = {
     'lxd': lxd.LXDPlatform,
+    'kvm': kvm.KVMPlatform,
 }
 
 
diff --git a/tests/cloud_tests/platforms/kvm.py b/tests/cloud_tests/platforms/kvm.py
new file mode 100644
index 0000000..70808f2
--- /dev/null
+++ b/tests/cloud_tests/platforms/kvm.py
@@ -0,0 +1,98 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base KVM platform."""
+import glob
+import os
+import shutil
+
+from simplestreams import filters
+from simplestreams import mirrors
+from simplestreams import objectstores
+from simplestreams import util as s_util
+
+from cloudinit import util as c_util
+from tests.cloud_tests.images import kvm as kvm_image
+from tests.cloud_tests.instances import kvm as kvm_instance
+from tests.cloud_tests.platforms import base
+from tests.cloud_tests import util
+
+
+class KVMPlatform(base.Platform):
+    """KVM test platform."""
+
+    platform_name = 'kvm'
+
+    def __init__(self, config):
+        """Set up platform."""
+        super(KVMPlatform, self).__init__(config)
+
+    def get_image(self, img_conf):
+        """Get image using specified image configuration.
+
+        @param img_conf: configuration for image
+        @return_value: cloud_tests.images instance
+        """
+        (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
+
+        filter = filters.get_filters(['arch=%s' % c_util.get_architecture(),
+                                      'release=%s' % img_conf['release'],
+                                      'ftype=disk1.img'])
+        mirror_config = {'filters': filter,
+                         'keep_items': False,
+                         'max_items': 1,
+                         'checksumming_reader': True,
+                         'item_download': True
+                         }
+
+        def policy(content, path):
+            return s_util.read_signed(content, keyring=img_conf['keyring'])
+
+        smirror = mirrors.UrlMirrorReader(url,
+                                          policy=policy)
+        tstore = objectstores.FileStore(img_conf['mirror_dir'])
+        tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
+                                             objectstore=tstore)
+        tmirror.sync(smirror, path)
+
+        search_d = os.path.join(img_conf['mirror_dir'], '**',
+                                img_conf['release'], '**', '*.img')
+
+        images = []
+        for fname in glob.iglob(search_d, recursive=True):
+            images.append(fname)
+
+        if len(images) != 1:
+            raise Exception('No unique images found')
+
+        image = kvm_image.KVMImage(self, img_conf, images[0])
+        if img_conf.get('override_templates', False):
+            image.update_templates(self.config.get('template_overrides', {}),
+                                   self.config.get('template_files', {}))
+        return image
+
+    def create_image(self, properties, config, features,
+                     src_img_path, image_desc=None, use_desc=None,
+                     user_data=None, meta_data=None):
+        """Create an image
+
+        @param src_image_path: image path to launch from
+        @param properties: image properties
+        @param config: image configuration
+        @param features: image features
+        @param image_desc: description of image being launched
+        @param use_desc: description of container's use
+        @return_value: cloud_tests.instances instance
+        """
+        name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc)
+        img_path = os.path.join(self.config['data_dir'], name + '.qcow2')
+        shutil.copy2(src_img_path, img_path)
+
+        # create copy of the latest image, return that as an instance
+        return kvm_instance.KVMInstance(self, img_path, properties, config,
+                                        features, user_data, meta_data)
+
+    def destroy(self):
+        """Clean up platform data."""
+        super(KVMPlatform, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml
index 45deb58..949266a 100644
--- a/tests/cloud_tests/releases.yaml
+++ b/tests/cloud_tests/releases.yaml
@@ -27,7 +27,12 @@ default_release_config:
         # features groups and additional feature settings
         feature_groups: []
         features: {}
-
+    kvm:
+        mirror_url: https://cloud-images.ubuntu.com/daily
+        mirror_dir: '/tmp/cloud_test_mirror'
+        keyring: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg
+        setup_overrides: null
+        override_templates: false
     # lxd specific default configuration options
     lxd:
         # default sstreams server to use for lxd image retrieval
@@ -121,6 +126,9 @@ releases:
         # EOL: Jul 2018
         default:
             enabled: true
+            release: artful
+            version: 17.10
+            family: ubuntu
             feature_groups:
                 - base
                 - debian_base
@@ -134,6 +142,9 @@ releases:
         # EOL: Jan 2018
         default:
             enabled: true
+            release: zesty
+            version: 17.04
+            family: ubuntu
             feature_groups:
                 - base
                 - debian_base
@@ -147,6 +158,9 @@ releases:
         # EOL: Jul 2017
         default:
             enabled: true
+            release: yakkety
+            version: 16.10
+            family: ubuntu
             feature_groups:
                 - base
                 - debian_base
@@ -160,6 +174,9 @@ releases:
         # EOL: Apr 2021
         default:
             enabled: true
+            release: xenial
+            version: 16.04
+            family: ubuntu
             feature_groups:
                 - base
                 - debian_base
@@ -173,6 +190,9 @@ releases:
         # EOL: Apr 2019
         default:
             enabled: true
+            release: trusty
+            version: 14.04
+            family: ubuntu
             feature_groups:
                 - base
                 - debian_base
diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py
index 8053a09..86a5f46 100644
--- a/tests/cloud_tests/setup_image.py
+++ b/tests/cloud_tests/setup_image.py
@@ -5,6 +5,7 @@
 from functools import partial
 import os
 
+from cloudinit import util as c_util
 from tests.cloud_tests import LOG
 from tests.cloud_tests import stage, util
 
@@ -191,6 +192,15 @@ def enable_repo(args, image):
     image.execute(['/bin/sh', '-c', cmd], description=msg)
 
 
+def generate_ssh_keys(args, image):
+    """Generate SSH keys to be used with image."""
+    LOG.info('generating SSH keys')
+    c_util.subp(['ssh-keygen', '-t', 'rsa', '-b', '4096',
+                 '-f', '%s/id_rsa' % args.data_dir, '-P', '',
+                 '-C', 'ubuntu@cloud_test'],
+                capture=True)
+
+
 def setup_image(args, image):
     """Set up image as specified in args.
 
@@ -226,6 +236,7 @@ def setup_image(args, image):
         'set up for {}'.format(image), calls, continue_after_error=False)
     LOG.debug('after setup complete, installed cloud-init version is: %s',
               installed_package_version(image, 'cloud-init'))
+    generate_ssh_keys(args, image)
     return res
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/snapshots/kvm.py b/tests/cloud_tests/snapshots/kvm.py
new file mode 100644
index 0000000..c15f834
--- /dev/null
+++ b/tests/cloud_tests/snapshots/kvm.py
@@ -0,0 +1,74 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Base KVM snapshot."""
+import os
+
+from tests.cloud_tests.snapshots import base
+
+
+class KVMSnapshot(base.Snapshot):
+    """KVM image copy backed snapshot."""
+
+    platform_name = "kvm"
+
+    def __init__(self, platform, properties, config, features,
+                 instance):
+        """Set up snapshot.
+
+        @param platform: platform object
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
+        """
+        self.instance = instance
+
+        super(KVMSnapshot, self).__init__(
+            platform, properties, config, features)
+
+    def launch(self, user_data, meta_data=None, block=True, start=True,
+               use_desc=None):
+        """Launch instance.
+
+        @param user_data: user-data for the instance
+        @param instance_id: instance-id for the instance
+        @param block: wait until instance is created
+        @param start: start instance and wait until fully started
+        @param use_desc: description of snapshot instance use
+        @return_value: an Instance
+        """
+        key_file = os.path.join(self.platform.config['data_dir'],
+                                self.platform.config['public_key'])
+        user_data = self.inject_ssh_key(user_data, key_file)
+
+        instance = self.platform.create_image(
+            self.properties, self.config, self.features,
+            self.instance.name, image_desc=str(self), use_desc=use_desc,
+            user_data=user_data, meta_data=meta_data)
+
+        if start:
+            instance.start()
+
+        return instance
+
+    def inject_ssh_key(self, user_data, key_file):
+        """Inject the authorized key into the user_data."""
+        with open(key_file) as f:
+            value = f.read()
+
+        key = 'ssh_authorized_keys:'
+        value = '  - %s' % value.strip()
+        user_data = user_data.split('\n')
+        if key in user_data:
+            user_data.insert(user_data.index(key) + 1, '%s' % value)
+        else:
+            user_data.insert(-1, '%s' % key)
+            user_data.insert(-1, '%s' % value)
+
+        return '\n'.join(user_data)
+
+    def destroy(self):
+        """Clean up snapshot data."""
+        self.instance.destroy()
+        super(KVMSnapshot, self).destroy()
+
+# vi: ts=4 expandtab
diff --git a/tools/xkvm b/tools/xkvm
new file mode 100755
index 0000000..a30ba91
--- /dev/null
+++ b/tools/xkvm
@@ -0,0 +1,664 @@
+#!/bin/bash
+
+set -f
+
+VERBOSITY=0
+KVM_PID=""
+DRY_RUN=false
+TEMP_D=""
+DEF_BRIDGE="virbr0"
+TAPDEVS=( )
+# OVS_CLEANUP gets populated with bridge:devname pairs used with ovs
+OVS_CLEANUP=( )
+MAC_PREFIX="52:54:00:12:34"
+KVM="kvm"
+declare -A KVM_DEVOPTS
+
+error() { echo "$@" 1>&2; }
+fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
+
+bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
+randmac() {
+    # return random mac addr within final 3 tokens
+    local random=""
+    random=$(printf "%02x:%02x:%02x" \
+        "$((${RANDOM}%256))" "$((${RANDOM}%256))" "$((${RANDOM}%256))")
+    padmac "$random"
+}
+
+cleanup() {
+    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
+    [ -z "${KVM_PID}" ] || kill "$KVM_PID"
+    if [ ${#TAPDEVS[@]} -ne 0 ]; then
+        local name item
+        for item in "${TAPDEVS[@]}"; do
+            [ "${item}" = "skip" ] && continue
+            debug 1 "removing" "$item"
+            name="${item%:*}"
+            if $DRY_RUN; then
+                error ip tuntap del mode tap "$name"
+            else
+                ip tuntap del mode tap "$name"
+            fi
+            [ $? -eq 0 ] || error "failed removal of $name"
+        done
+        if [ ${#OVS_CLEANUP[@]} -ne 0 ]; then
+            # with linux bridges, there seems to be no harm in just deleting
+            # the device (not detaching from the bridge).  However, with
+            # ovs, you have to remove them from the bridge, or later it
+            # will refuse to add the same name.
+            error "cleaning up ovs ports: ${OVS_CLEANUP[@]}"
+            if ${DRY_RUN}; then
+                error sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}"
+            else
+                sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}"
+            fi
+        fi
+    fi
+}
+
+debug() {
+    local level=${1}; shift;
+    [ "${level}" -gt "${VERBOSITY}" ] && return
+    error "${@}"
+}
+
+Usage() {
+    cat <<EOF
+Usage: ${0##*/} [ options ] -- kvm-args [ ... ]
+
+   run kvm with a tap interface.
+
+   options:
+      -n | --netdev NETDEV    netdev can be 'user' or a bridge.
+                              default is to bridge to $DEF_BRIDGE
+      -d | --disk  DISK.img   attach DISK.img as a disk (via virtio)
+           --dry-run          only report what would be done
+
+           --uefi             boot with efi
+           --uefi-nvram=FILE  boot with efi, using nvram settings in FILE
+                              if FILE not present, copy from defaults.
+
+   NETDEV:
+    Above, 'NETDEV' is a comma delimited string
+    The first field must be
+     * bridge name: (br0 or virbr0): attach a device to this bridge
+     * literal 'user': use qemu user networking
+
+    Additional fields are optional, and can be anything that is acceptable
+    to kvm either as:
+      * '-device virtio-net-pci' option (see 'kvm -device virtio-net-pci,?')
+      * '-net [user|tap]' option
+
+   Example:
+     * xkvm --netdev br0,macaddr=:05 -- -drive file=disk.img,if=virtio -curses
+       attach a tap device to bridge 'br0' with mac address
+         '${MAC_PREFIX}:05'
+
+     * xkvm --netdev user,mac=random --netdev br1,model=e1000,mac=auto -- -curses
+       attach virtio user networking nic with random mac address
+       attach tap device to br1 bridge as e1000 with unspecified mac
+
+     * xkvm --disk disk1.img
+EOF
+}
+
+isdevopt() {
+    local model="$1" input="${2%%=*}"
+    local out="" opt="" opts=()
+    if [ -z "${KVM_DEVOPTS[$model]}" ]; then
+        out=$($KVM -device "$model,?" 2>&1) &&
+            out=$(echo "$out" | sed -e "s,[^.]*[.],," -e 's,=.*,,') &&
+            KVM_DEVOPTS[$model]="$out" ||
+            { error "bad device model $model?"; exit 1; }
+    fi
+    opts=( ${KVM_DEVOPTS[$model]} )
+    for opt in "${opts[@]}"; do
+        [ "$input" = "$opt" ] && return 0
+    done
+    return 1
+}
+
+padmac() {
+    # return a full mac, given a subset.
+    # assume whatever is input is the last portion to be
+    # returned, and fill it out with entries from MAC_PREFIX
+    local mac="$1" num="$2" prefix="${3:-$MAC_PREFIX}" itoks="" ptoks=""
+    # if input is empty set to :$num
+    [ -n "$mac" ] || mac=$(printf "%02x" "$num") || return
+    itoks=( ${mac//:/ } )
+    ptoks=( ${prefix//:/ } )
+    rtoks=( )
+    for r in ${ptoks[@]:0:6-${#itoks[@]}} ${itoks[@]}; do
+        rtoks[${#rtoks[@]}]="0x$r"
+    done
+    _RET=$(printf "%02x:%02x:%02x:%02x:%02x:%02x" "${rtoks[@]}")
+}
+
+make_nics_Usage() {
+    cat <<EOF
+Usage: ${0##*/} tap-control make-nics [options] bridge [bridge [..]]
+
+   make a tap device on each of bridges requested
+   outputs: 'tapname:type' for each input, or 'skip' if nothing needed.
+
+   type is one of 'brctl' or 'ovs'
+EOF
+}
+
+make_nics() {
+    # takes input of list of bridges to create a tap device on
+    # and echos either 'skip' or
+    # <tapname>:<type> for each tap created
+    # type is one of "ovs" or "brctl"
+    local short_opts="v"
+    local long_opts="--verbose"
+    local getopt_out=""
+    getopt_out=$(getopt --name "${0##*/} make-nics" \
+        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
+        eval set -- "${getopt_out}" || { make_nics_Usage 1>&2; return 1; }
+
+    local cur="" next=""
+    while [ $# -ne 0 ]; do
+        cur=${1}; next=${2};
+        case "$cur" in
+            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
+            --) shift; break;;
+        esac
+        shift;
+    done
+
+    [ $# -ne 0 ] || {
+        make_nics_Usage 1>&2; error "must give bridge";
+        return 1;
+    }
+
+    local owner="" ovsbrs="" tap="" tapnum="0" brtype="" bridge=""
+    [ "$(id -u)" = "0" ] || { error "must be root for make-nics"; return 1; }
+    owner="${SUDO_USER:-root}"
+    ovsbrs=""
+    if command -v ovs-vsctl >/dev/null 2>&1; then
+        out=$(ovs-vsctl list-br)
+        out=$(echo "$out" | sed "s/\n/,/")
+        ovsbrs=",$out,"
+    fi
+    for bridge in "$@"; do
+        [ "$bridge" = "user" ] && echo skip && continue
+        [ "${ovsbrs#*,${bridge},}" != "$ovsbrs" ] &&
+            btype="ovs" || btype="brctl"
+        tapnum=0;
+        while [ -e /sys/class/net/tapvm$tapnum ]; do tapnum=$(($tapnum+1)); done
+        tap="tapvm$tapnum"
+        debug 1 "creating $tap:$btype on $bridge" 1>&2
+        ip tuntap add mode tap user "$owner" "$tap" ||
+            { error "failed to create tap '$tap' for '$owner'"; return 1; }
+        ip link set "$tap" up 1>&2 || {
+            error "failed to bring up $tap";
+            ip tuntap del mode tap "$tap";
+            return 1;
+        }
+        if [ "$btype" = "ovs" ]; then
+            ovs-vsctl add-port "$bridge" "$tap" 1>&2 || {
+                error "failed: ovs-vsctl add-port $bridge $tap";
+                ovs-vsctl del-port "$bridge" "$tap"
+                return 1;
+            }
+        else
+            ip link set "$tap" master "$bridge" 1>&2 || {
+                error "failed to add tap '$tap' to '$bridge'"
+                ip tuntap del mode tap "$tap";
+                return 1
+            }
+        fi
+        echo "$tap:$btype"
+    done
+}
+
+ovs_cleanup() {
+    [ "$(id -u)" = "0" ] ||
+        { error "must be root for ovs-cleanup"; return 1; }
+    local item="" errors=0
+    # TODO: if get owner (SUDO_USERNAME) and if that isn't
+    # the owner, then do not delete.
+    for item in "$@"; do
+        name=${item#*:}
+        bridge=${item%:*}
+        ovs-vsctl del-port "$bridge" "$name" || errors=$((errors+1))
+    done
+    return $errors
+}
+
+quote_cmd() {
+    local quote='"' x="" vline=""
+    for x in "$@"; do
+        if [ "${x#* }" != "${x}" ]; then
+            if [ "${x#*$quote}" = "${x}" ]; then
+                x="\"$x\""
+            else
+                x="'$x'"
+            fi
+        fi
+        vline="${vline} $x"
+    done
+    echo "$vline"
+}
+
+get_bios_opts() {
+    # get_bios_opts(bios, uefi, nvram)
+    # bios is a explicit bios to boot.
+    # uefi is boolean indicating uefi
+    # nvram is optional and indicates that ovmf vars should be copied
+    # to that file if it does not exist. if it exists, use it.
+    local bios="$1" uefi="${2:-false}" nvram="$3"
+    local ovmf_dir="/usr/share/OVMF"
+    local bios_opts="" pflash_common="if=pflash,format=raw"
+    unset _RET
+    _RET=( )
+    if [ -n "$bios" ]; then
+        _RET=( -drive "${pflash_common},file=$bios" )
+        return 0
+    elif ! $uefi; then
+        return 0
+    fi
+
+    # ovmf in older releases (14.04) shipped only a single file
+    #   /usr/share/ovmf/OVMF.fd
+    # newer ovmf ships split files
+    #   /usr/share/OVMF/OVMF_CODE.fd
+    #   /usr/share/OVMF/OVMF_VARS.fd
+    # with single file, pass only one file and read-write
+    # with split, pass code as readonly and vars as read-write
+    local joined="/usr/share/ovmf/OVMF.fd"
+    local code="/usr/share/OVMF/OVMF_CODE.fd"
+    local vars="/usr/share/OVMF/OVMF_VARS.fd"
+    local split="" nvram_src=""
+    if [ -e "$code" -o -e "$vars" ]; then
+        split=true
+        nvram_src="$vars"
+    elif [ -e "$joined" ]; then
+        split=false
+        nvram_src="$joined"
+    elif [ -n "$nvram" -a -e "$nvram" ]; then
+        error "WARN: nvram given, but did not find expected ovmf files."
+        error "      assuming this is code and vars (OVMF.fd)"
+        split=false
+    else
+        error "uefi support requires ovmf bios: apt-get install -qy ovmf"
+        return 1
+    fi
+
+    if [ -n "$nvram" ]; then
+        if [ ! -f "$nvram" ]; then
+            cp "$nvram_src" "$nvram" || 
+                { error "failed copy $nvram_src to $nvram"; return 1; }
+            debug 1 "copied $nvram_src to $nvram"
+        fi
+    else
+        debug 1 "uefi without --uefi-nvram storage." \
+            "nvram settings likely will not persist."
+        nvram="${nvram_src}"
+    fi
+
+    if [ ! -w "$nvram" ]; then
+        debug 1 "nvram file ${nvram} is readonly"
+        nvram_ro="readonly"
+    fi
+
+    if $split; then
+        # to ensure bootability firmware must be first, then variables
+        _RET=( -drive "${pflash_common},file=$code,readonly" )
+    fi
+    _RET=( "${_RET[@]}"
+           -drive "${pflash_common},file=$nvram${nvram_ro:+,${nvram_ro}}" )
+}
+
+main() {
+    local short_opts="hd:n:v"
+    local long_opts="bios:,help,dowait,disk:,dry-run,kvm:,no-dowait,netdev:,uefi,uefi-nvram:,verbose"
+    local getopt_out=""
+    getopt_out=$(getopt --name "${0##*/}" \
+        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
+        eval set -- "${getopt_out}" || { bad_Usage; return 1; }
+
+    local bridge="$DEF_BRIDGE" oifs="$IFS"
+    local netdevs="" need_tap="" ret="" p="" i="" pt="" cur="" conn=""
+    local kvm="" kvmcmd="" archopts=""
+    local def_disk_driver=${DEF_DISK_DRIVER:-"virtio-blk"}
+    local def_netmodel=${DEF_NETMODEL:-"virtio-net-pci"}
+    local bios="" uefi=false uefi_nvram=""
+
+    archopts=( )
+    kvmcmd=( )
+    netdevs=( )
+    addargs=( )
+    diskdevs=( )
+    diskargs=( )
+
+    # dowait: run qemu-system with a '&' and then 'wait' on the pid.
+    #  the reason to do this or not do this has to do with interactivity
+    #  if detached with &, then user input will not go to xkvm.
+    #  if *not* detached, then signal handling is blocked until
+    #  the foreground subprocess returns. which means we can't handle
+    #  a sigterm and kill the qemu-system process.
+    #  We default to dowait=false if input and output are a terminal
+    local dowait=""
+    [ -t 0 -a -t 1 ] && dowait=false || dowait=true
+    while [ $# -ne 0 ]; do
+        cur=${1}; next=${2};
+        case "$cur" in
+            -h|--help) Usage; exit 0;;
+            -d|--disk)
+                diskdevs[${#diskdevs[@]}]="$next"; shift;;
+            --dry-run) DRY_RUN=true;;
+            --kvm) kvm="$next"; shift;;
+            -n|--netdev)
+                netdevs[${#netdevs[@]}]=$next; shift;;
+            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
+            --dowait) dowait=true;;
+            --no-dowait) dowait=false;;
+            --bios) bios="$next"; shift;;
+            --uefi) uefi=true;;
+            --uefi-nvram) uefi=true; uefi_nvram="$next"; shift;;
+            --) shift; break;;
+        esac
+        shift;
+    done
+
+    [ ${#netdevs[@]} -eq 0 ] && netdevs=( "${DEF_BRIDGE}" )
+    pt=( "$@" )
+
+    local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci"
+    [ -n "$kvm" ] && kvm_pkg="none"
+    case $(uname -m) in
+        i?86)
+            [ -n "$kvm" ] ||
+                { kvm="qemu-system-i386"; kvm_pkg="qemu-system-x86"; }
+            ;;
+        x86_64)
+            [ -n "$kvm" ] ||
+                { kvm="qemu-system-x86_64"; kvm_pkg="qemu-system-x86"; }
+            ;;
+        s390x)
+            [ -n "$kvm" ] ||
+                { kvm="qemu-system-s390x"; kvm_pkg="qemu-system-misc"; }
+            def_netmodel=${DEF_NETMODEL:-"virtio-net-ccw"}
+            virtio_scsi_bus="virtio-scsi-ccw"
+            ;;
+        ppc64*)
+            [ -n "$kvm" ] ||
+                { kvm="qemu-system-ppc64"; kvm_pkg="qemu-system-ppc"; }
+            def_netmodel="virtio-net-pci"
+            # virtio seems functional on in 14.10, but might want scsi here
+            #def_diskif="scsi"
+            archopts=( "${archopts[@]}" -machine pseries,usb=off )
+            archopts=( "${archopts[@]}" -device spapr-vscsi )
+            ;;
+        *) kvm=qemu-system-$(uname -m);;
+    esac
+    KVM="$kvm"
+    kvmcmd=( $kvm -enable-kvm )
+
+    local bios_opts=""
+    if [ -n "$bios" ] && $uefi; then
+        error "--uefi (or --uefi-nvram) is incompatible with --bios"
+        return 1
+    fi
+    get_bios_opts "$bios" "$uefi" "$uefi_nvram" ||
+        { error "failed to get bios opts"; return 1; }
+    bios_opts=( "${_RET[@]}" )
+
+    local out="" fmt="" bus="" unit="" index="" serial="" driver="" devopts=""
+    local busorindex="" driveopts="" cur="" val="" file=""
+    for((i=0;i<${#diskdevs[@]};i++)); do
+        cur=${diskdevs[$i]}
+        IFS=","; set -- $cur; IFS="$oifs"
+        driver=""
+        id=$(printf "disk%02d" "$i")
+        file=""
+        fmt=""
+        bus=""
+        unit=""
+        index=""
+        serial=""
+        for tok in "$@"; do
+            [ "${tok#*=}" = "${tok}" -a -f "${tok}" -a -z "$file" ] && file="$tok"
+            val=${tok#*=}
+            case "$tok" in
+                driver=*) driver=$val;;
+                if=virtio) driver=virtio-blk;;
+                if=scsi) driver=scsi-hd;;
+                if=pflash) driver=;;
+                if=sd|if=mtd|floppy) fail "do not know what to do with $tok on $cur";;
+                id=*) id=$val;;
+                file=*) file=$val;;
+                fmt=*|format=*) fmt=$val;;
+                serial=*) serial=$val;;
+                bus=*) bus=$val;;
+                unit=*) unit=$val;;
+                index=*) index=$val;;
+            esac
+        done
+        [ -z "$file" ] && fail "did not read a file from $cur"
+        if [ -f "$file" -a -z "$fmt" ]; then
+            out=$(LANG=C qemu-img info "$file") &&
+                fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') ||
+                { error "failed to determine format of $file"; return 1; }
+        else
+            fmt=raw
+        fi
+        if [ -z "$driver" ]; then
+            driver="$def_disk_driver"
+        fi
+        if [ -z "$serial" ]; then
+            serial="${file##*/}"
+        fi
+
+        # make sure we add either bus= or index=
+        if [ -n "$bus" -o "$unit" ] && [ -n "$index" ]; then
+            fail "bus and index cant be specified together: $cur"
+        elif [ -z "$bus" -a -z "$unit" -a -z "$index" ]; then
+            index=$i
+        elif [ -n "$bus" -a -z "$unit" ]; then
+            unit=$i
+        fi
+
+        busorindex="${bus:+bus=$bus,unit=$unit}${index:+index=${index}}"
+        diskopts="file=${file},id=$id,if=none,format=$fmt,$busorindex"
+        devopts="$driver,drive=$id${serial:+,serial=${serial}}"
+        for tok in "$@"; do
+            case "$tok" in
+                id=*|if=*|driver=*|$file|file=*) continue;;
+                fmt=*|format=*) continue;;
+                serial=*|bus=*|unit=*|index=*) continue;;
+            esac
+            isdevopt "$driver" "$tok" && devopts="${devopts},$tok" ||
+                diskopts="${diskopts},${tok}"
+        done
+
+        diskargs=( "${diskargs[@]}" -drive "$diskopts" -device "$devopts" )
+    done
+
+    local mnics_vflag=""
+    for((i=0;i<${VERBOSITY}-1;i++)); do mnics_vflag="${mnics_vflag}v"; done
+    [ -n "$mnics_vflag" ] && mnics_vflag="-${mnics_vflag}"
+
+    # now go through and split out options
+    # -device virtio-net-pci,netdev=virtnet0,mac=52:54:31:15:63:02
+    # -netdev type=tap,id=virtnet0,vhost=on,script=/etc/kvm/kvm-ifup.br0,downscript=no
+    local netopts="" devopts="" id="" need_taps=0 model=""
+    local device_args netdev_args
+    device_args=( )
+    netdev_args=( )
+    connections=( )
+    for((i=0;i<${#netdevs[@]};i++)); do
+        id=$(printf "net%02d" "$i")
+        netopts="";
+        devopts=""
+        # mac=auto is 'unspecified' (let qemu assign one)
+        mac="auto"
+        #vhost="off"
+
+        IFS=","; set -- ${netdevs[$i]}; IFS="$oifs"
+        bridge=$1; shift;
+        if [ "$bridge" = "user" ]; then
+            netopts="type=user"
+            ntype="user"
+            connections[$i]="user"
+        else
+            need_taps=1
+            ntype="tap"
+            netopts="type=tap"
+            connections[$i]="$bridge"
+        fi
+        netopts="${netopts},id=$id"
+        [ "$ntype" = "tap" ] && netopts="${netopts},script=no,downscript=no"
+
+        model="${def_netmodel}"
+        for tok in "$@"; do
+            [ "${tok#model=}" = "${tok}" ] && continue
+            case "${tok#model=}" in
+                virtio) model=virtio-net-pci;;
+                *) model=${tok#model=};;
+            esac
+        done
+
+        for tok in "$@"; do
+            case "$tok" in
+                mac=*) mac="${tok#mac=}"; continue;;
+                macaddr=*) mac=${tok#macaddr=}; continue;;
+                model=*) continue;;
+            esac
+
+            isdevopt "$model" "$tok" && devopts="${devopts},$tok" ||
+                netopts="${netopts},${tok}"
+        done
+        devopts=${devopts#,}
+        netopts=${netopts#,}
+
+        if [ "$mac" != "auto" ]; then
+            [ "$mac" = "random" ] && randmac && mac="$_RET"
+            padmac "$mac" "$i"
+            devopts="${devopts:+${devopts},}mac=$_RET"
+        fi
+        devopts="$model,netdev=$id${devopts:+,${devopts}}"
+        #netopts="${netopts},vhost=${vhost}"
+
+        device_args[$i]="$devopts"
+        netdev_args[$i]="$netopts"
+    done
+
+    trap cleanup EXIT
+
+    reqs=( "$kvm" )
+    pkgs=( "$kvm_pkg" )
+    for((i=0;i<${#reqs[@]};i++)); do
+        req=${reqs[$i]}
+        pkg=${pkgs[$i]}
+        [ "$pkg" = "none" ] && continue
+        command -v "$req" >/dev/null || {
+            missing="${missing:+${missing} }${req}"
+            missing_pkgs="${missing_pkgs:+${missing_pkgs} }$pkg"
+        }
+    done
+    if [ -n "$missing" ]; then
+        local reply cmd=""
+        cmd=( sudo apt-get --quiet install ${missing_pkgs} )
+        error "missing prereqs: $missing";
+        error "install them now with the following?: ${cmd[*]}"
+        read reply && [ "$reply" = "y" -o "$reply" = "Y" ] ||
+            { error "run: apt-get install ${missing_pkgs}"; return 1; }
+        "${cmd[@]}" || { error "failed to install packages"; return 1; }
+    fi
+
+    if [ $need_taps -ne 0 ]; then
+        local missing="" missing_pkgs="" reqs="" req="" pkgs="" pkg=""
+        for i in "${connections[@]}"; do
+            [ "$i" = "user" -o -e "/sys/class/net/$i" ] ||
+                missing="${missing} $i"
+        done
+        [ -z "$missing" ] || {
+            error "cannot create connection on: ${missing# }."
+            error "bridges do not exist.";
+            return 1;
+        }
+        error "creating tap devices: ${connections[*]}"
+        if $DRY_RUN; then
+            error "sudo $0 tap-control make-nics" \
+                $mnics_vflag "${connections[@]}"
+            taps=""
+            for((i=0;i<${#connections[@]};i++)); do
+                if [ "${connections[$i]}" = "user" ]; then
+                    taps="${taps} skip"
+                else
+                    taps="${taps} dryruntap$i:brctl"
+                fi
+            done
+        else
+            taps=$(sudo "$0" tap-control make-nics \
+                   ${mnics_vflag} "${connections[@]}") ||
+                { error "$failed to make-nics ${connections[*]}"; return 1; }
+        fi
+        TAPDEVS=( ${taps} )
+        for((i=0;i<${#TAPDEVS[@]};i++)); do
+            cur=${TAPDEVS[$i]}
+            [ "${cur#*:}" = "ovs" ] || continue
+            conn=${connections[$i]}
+            OVS_CLEANUP[${#OVS_CLEANUP[@]}]="${conn}:${cur%:*}"
+        done
+
+        debug 2 "tapdevs='${TAPDEVS[@]}'"
+        [ ${#OVS_CLEANUP[@]} -eq 0 ] || error "OVS_CLEANUP='${OVS_CLEANUP[*]}'"
+
+        for((i=0;i<${#TAPDEVS[@]};i++)); do
+            cur=${TAPDEVS[$i]}
+            [ "$cur" = "skip" ] && continue
+            netdev_args[$i]="${netdev_args[$i]},ifname=${cur%:*}";
+        done
+    fi
+
+    netargs=()
+    for((i=0;i<${#device_args[@]};i++)); do
+        netargs=( "${netargs[@]}" -device "${device_args[$i]}"
+                  -netdev "${netdev_args[$i]}")
+    done
+
+    local bus_devices
+    bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" )
+    cmd=( "${kvmcmd[@]}" "${archopts[@]}" 
+          "${bios_opts[@]}"
+          "${bus_devices[@]}"
+          "${netargs[@]}"
+          "${diskargs[@]}" "${pt[@]}" )
+    local pcmd=$(quote_cmd "${cmd[@]}")
+    error "$pcmd"
+    ${DRY_RUN} && return 0
+
+    if $dowait; then
+        "${cmd[@]}" &
+        KVM_PID=$!
+        debug 1 "kvm pid=$KVM_PID. my pid=$$"
+        wait
+        ret=$?
+        KVM_PID=""
+    else
+        "${cmd[@]}"
+        ret=$?
+    fi
+    return $ret
+}
+
+
+if [ "$1" = "tap-control" ]; then
+    shift
+    mode=$1
+    shift || fail "must give mode to tap-control"
+    case "$mode" in
+        make-nics) make_nics "$@";;
+        ovs-cleanup) ovs_cleanup "$@";;
+        *) fail "tap mode must be either make-nics or ovs-cleanup";;
+    esac
+else
+    main "$@"
+fi
+
+# vi: ts=4 expandtab