diff --git a/NEWS b/NEWS index 3d0a93a5dd..592a672abd 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ Flocker 1.4.0.dev1 (2015-09-08) =============================== - Improved the error handling when Cinder volumes enter an unexpected state. (FLOC-2970) +- ``run-acceptance-tests`` now supports all OpenStack variants, not just RackSpace. (FLOC-3017) Flocker 1.3.0 (2015-09-04) ===================================== diff --git a/admin/acceptance.py b/admin/acceptance.py index a9cff1dc9d..1e330ebeb4 100644 --- a/admin/acceptance.py +++ b/admin/acceptance.py @@ -791,6 +791,34 @@ def _runner_RACKSPACE(self, package_source, dataset_backend, package_source, dataset_backend, "rackspace", provider_config ) + def _runner_OPENSTACK(self, package_source, dataset_backend, + provider_config): + """ + :param PackageSource package_source: The source of omnibus packages. + :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. + :param provider_config: The ``openstack`` section of the acceptance + testing configuration file. The section of the configuration + file should look something like: + + openstack: + auth_plugin: + auth_url: + region: + tenant: + username: + secret: + keyname: + images: + ubuntu-14.04: + centos-7: # noqa + flavor: + + :see: :ref:`acceptance-testing-openstack-config` + """ + return self._libcloud_runner( + package_source, dataset_backend, "openstack", provider_config + ) + def _runner_AWS(self, package_source, dataset_backend, provider_config): """ diff --git a/docs/gettinginvolved/acceptance-testing.rst b/docs/gettinginvolved/acceptance-testing.rst index 0c2f6d8190..03364f8d1d 100644 --- a/docs/gettinginvolved/acceptance-testing.rst +++ b/docs/gettinginvolved/acceptance-testing.rst @@ -162,6 +162,47 @@ Rackspace can use these dataset backends: admin/run-acceptance-tests --distribution centos-7 --provider rackspace --config-file config.yml +.. _acceptance-testing-openstack-config: + +OpenStack +~~~~~~~~~ + +To run the acceptance tests on OpenStack, you need: + +- An OpenStack account and the associated password or API key. +- The URL of the OpenStack keystone service. +- An ssh-key registered with the OpenStack account. +- The OpenStack image name or image ID for images containing supported Flocker node operating systems. +- The OpenStack flavor to use for acceptance test nodes. + +To use the OpenStack provider, the configuration file should include an item like: + +.. code-block:: yaml + + openstack: + auth_plugin: + auth_url: + region: + tenant: + username: + secret: + keyname: + images: + ubuntu-14.04: + centos-7: # noqa + flavor: + +You will need a ssh agent running with access to the corresponding private key. + +Openstack can use these dataset backends: + + * :ref:`OpenStack`. + * :ref:`ZFS`. + * :ref:`Loopback`. + +.. prompt:: bash $ + + admin/run-acceptance-tests --distribution centos-7 --provider openstack --config-file config.yml .. _acceptance-testing-aws-config: diff --git a/docs/gettinginvolved/example-acceptance.yml b/docs/gettinginvolved/example-acceptance.yml index dc8a3f335a..41de236a3a 100644 --- a/docs/gettinginvolved/example-acceptance.yml +++ b/docs/gettinginvolved/example-acceptance.yml @@ -14,6 +14,19 @@ rackspace: key: "33333333333333333333333333333333" keyname: "your.rackspace.ssh.key.identifier" +openstack: + auth_plugin: + auth_url: + region: + tenant: + username: + secret: + keyname: + images: + ubuntu-14.04: + centos-7: # noqa + flavor: + managed: addresses: - "192.0.2.15" diff --git a/flocker/provision/__init__.py b/flocker/provision/__init__.py index 13452f693b..3f4b1a5d97 100644 --- a/flocker/provision/__init__.py +++ b/flocker/provision/__init__.py @@ -7,11 +7,13 @@ from ._common import PackageSource, Variants from ._install import provision, configure_cluster from ._rackspace import rackspace_provisioner +from ._openstack import openstack_provisioner from ._aws import aws_provisioner from ._ca import Certificates CLOUD_PROVIDERS = { 'rackspace': rackspace_provisioner, + 'openstack': openstack_provisioner, 'aws': aws_provisioner, } diff --git a/flocker/provision/_libcloud.py b/flocker/provision/_libcloud.py index 1b4e2efbb9..5c1c071ba6 100644 --- a/flocker/provision/_libcloud.py +++ b/flocker/provision/_libcloud.py @@ -12,16 +12,16 @@ from flocker.provision._ssh import run_remotely, run_from_args -def get_size(driver, size_id): +def get_size(driver, size_name_or_id): """ Return a ``NodeSize`` corresponding to a given id. :param driver: The libcloud driver to query for sizes. """ try: - return [s for s in driver.list_sizes() if s.id == size_id][0] + return [s for s in driver.list_sizes() if size_name_or_id in (s.id, s.name)][0] except IndexError: - raise ValueError("Unknown size.", size_id) + raise ValueError("Unknown size.", size_name_or_id) def get_image(driver, image_name): @@ -224,7 +224,13 @@ def create_node(self, name, distribution, ) node, addresses = self._driver.wait_until_running( - [node], wait_period=15)[0] + [node], + wait_period=15, + # XXX: devstack guests don't have public_ips by default. + # Need to figure out how to handle this generally. Look at + # self.use_private_addresses + ssh_interface="private_ips", + )[0] public_address = addresses[0] diff --git a/flocker/provision/_openstack.py b/flocker/provision/_openstack.py new file mode 100644 index 0000000000..bc95d7d8a8 --- /dev/null +++ b/flocker/provision/_openstack.py @@ -0,0 +1,154 @@ +# Copyright ClusterHQ Inc. See LICENSE file for details. + +""" +OpenStack provisioner. +""" + +from textwrap import dedent +from time import time + +from effect.retry import retry +from effect import Effect, Constant + +from ._libcloud import LibcloudProvisioner +from ._install import ( + provision, + task_install_ssh_key, + task_open_control_firewall, +) +from ._ssh import run_remotely + +from ._effect import sequence + +# XXX: Copied from _aws. Needs refactoring +_usernames = { + 'centos-7': 'centos', + 'ubuntu-14.04': 'ubuntu', + 'ubuntu-15.04': 'ubuntu', +} + + +def get_default_username(distribution): + """ + Return the username available by default on a system. + + :param str distribution: Name of the operating system distribution + :return str: The username made available by AWS for this distribution. + """ + return _usernames[distribution] + + +def provision_openstack(node, package_source, distribution, variants): + """ + Provision flocker on this node. + + :param LibcloudNode node: Node to provision. + :param PackageSource package_source: See func:`task_install_flocker` + :param bytes distribution: See func:`task_install_flocker` + :param set variants: The set of variant configurations to use when + provisioning + """ + username = get_default_username(distribution) + commands = [] + # cloud-init may not have allowed sudo without tty yet, so try SSH key + # installation for a few more seconds: + start = [] + + # XXX Copied from _aws.py -- refactor. + def for_ten_seconds(*args, **kwargs): + if not start: + start.append(time()) + return Effect(Constant((time() - start[0]) < 30)) + + commands.append(run_remotely( + username=username, + address=node.address, + commands=retry(task_install_ssh_key(), for_ten_seconds), + )) + + commands.append(run_remotely( + username='root', + address=node.address, + commands=provision( + package_source=package_source, + distribution=node.distribution, + variants=variants, + ), + )) + + commands.append(run_remotely( + username='root', + address=node.address, + commands=sequence([ + provision( + package_source=package_source, + distribution=node.distribution, + variants=variants, + ), + # https://clusterhq.atlassian.net/browse/FLOC-1550 + # This should be part of ._install.configure_cluster + # task_open_control_firewall(node.distribution), + ]), + )) + + return sequence(commands) + + +def openstack_provisioner(auth_url, auth_plugin, username, secret, region, + keyname, images, flavor, tenant): + """ + Create a LibCloudProvisioner for provisioning nodes on openstack. + + :param bytes auth_url: The keystone URL. + :param bytes auth_plugin: The OpenStack authentication mechanism. One of + password or apikey. + :param bytes username: The user to connect to with. + :param bytes secret: The password or API key associated with the user. + :param bytes region: The region in which to launch the instance. + :param bytes keyname: The name of an existing ssh public key configured in + openstack. The provision step assumes the corresponding private key is + available from an agent. + :param dict images: A mapping of supported operating systems to a + corresponding OpenStack image name or image ID. + :param bytes flavor: A flavor name or flavor ID available in the target + OpenStack installation. + :param bytes tenant: The name of an OpenStack tenant or project. + """ + # Import these here, so that this can be imported without + # installng libcloud. + from libcloud.compute.providers import get_driver, Provider + + # LibCloud chooses OpenStack auth plugins using a weird naming scheme. + # See https://libcloud.readthedocs.org/en/latest/compute/drivers/openstack.html#connecting-to-the-openstack-installation # noqa + auth_versions = { + "apikey": "2.0_apikey", + "password": "2.0_password", + } + # See https://libcloud.readthedocs.org/en/latest/compute/drivers/openstack.html # noqa + driver = get_driver(Provider.OPENSTACK)( + key=username, + secret=secret, + region=region, + ex_force_auth_url=auth_url, + ex_force_auth_version=auth_versions[auth_plugin], + ex_force_service_region=region, + ex_tenant_name=tenant, + ) + + provisioner = LibcloudProvisioner( + driver=driver, + keyname=keyname, + image_names=images, + create_node_arguments=lambda **kwargs: { + "ex_config_drive": "true", + "ex_userdata": dedent("""\ + #!/bin/sh + sed -i '/Defaults *requiretty/d' /etc/sudoers + """), + }, + provision=provision_openstack, + default_size=flavor, + get_default_user=get_default_username, + ) + + return provisioner