From 34a7a72a85ba9cf867a14ad353d15fbb28562e7b Mon Sep 17 00:00:00 2001 From: pceicicd Date: Tue, 3 Oct 2023 21:01:37 +0000 Subject: [PATCH] Add support to programmably start a profile on the POWDER testbed, which can in turn to be used in tests Change-Id: I17d5ca0eea522ffae985e15fe5a5301f4b358965 Signed-off-by: pceicicd --- XTesting/powder-control/README.md | 74 +++++++++++ XTesting/powder-control/powder/experiment.py | 185 +++++++++++++++++++++++++++ XTesting/powder-control/powder/rpc.py | 124 ++++++++++++++++++ XTesting/powder-control/requirements.txt | 2 + XTesting/powder-control/start-profile.py | 83 ++++++++++++ 5 files changed, 468 insertions(+) create mode 100644 XTesting/powder-control/README.md create mode 100644 XTesting/powder-control/powder/experiment.py create mode 100644 XTesting/powder-control/powder/rpc.py create mode 100644 XTesting/powder-control/requirements.txt create mode 100755 XTesting/powder-control/start-profile.py diff --git a/XTesting/powder-control/README.md b/XTesting/powder-control/README.md new file mode 100644 index 0000000..4baa006 --- /dev/null +++ b/XTesting/powder-control/README.md @@ -0,0 +1,74 @@ +# About + +This project shows how to instantiate a Powder experiment +programmatically via the Powder portal API. + +In [powder/experiment.py](powder/experiment.py), there is a class +`PowderExperiment`, which serves as the main abstraction for starting, +interacting with, and terminating a single Powder experiment. It relies on +[powder/rpc.py](powder/rpc.py) for interacting with the Powder RPC server + +In [start-profile.py](start-profile.py), you can see how one might use it +to instantiate a Powder experiment by specifying a given profile name + +## Getting Started + +In order to run [start-profile.py](start-profile.py) or otherwise use the tools in this +project, you'll need a Powder account to which you've added your public `ssh` +key. If you haven't already done this, you can find instructions +[here](https://docs.powderwireless.net/users.html#%28part._ssh-access%29). +You'll also need to download your Powder credentials. You'll find a button to do +so in the drop-down menu accessed by clicking on your username after logging +into the Powder portal. This will download a file called `cloudlab.pem`, which +you will need later. + +You will need to make sure the machine you are using has `python3` installed, as well as +the packages in [requirements.txt](requirements.txt). You can install the +packages by doing + +```bash +pip install -r requirements.txt +``` + +Whatever machine you are using to run [start-profile.py](start-profile.py), it will need a +local copy of the `cloudlab.pem` file you downloaded earlier. + +### Running the Example + +The RPC client that `PowderExperiment` relies on to interact with the Powder RPC +server expects some environment variables to be set. If your private `ssh` key +is encrypted, the key password needs to be set in an environment variable as +well, unless you have already started `ssh-agent`. + +If your ssh key is encrypted: + +```bash +set +o history +PROJECT=your_project_name PROFILE=your_profile_name \ +USER=your_powder_username PWORD=your_powder_password \ +CERT=/path/to/your/cloulab.pem KEYPWORD=your_ssh_key_password ./start-profile.py +``` + +If not: + +```bash +set +o history +PROJECT=your_project_name PROFILE=your_profile_name \ +USER=your_powder_username PWORD=your_powder_password \ +CERT=/path/to/your/cloulab.pem ./start-profile.py +``` + +The `set +o history` command will keep these passwords out of your `bash` +history (assuming you're using `bash`). The PROJECT and PROFILE environment +variables are optional, as by default it will use the 'osc' project which is +dedicated to the OSC community usage, and 'ubuntu-20' profile for the widely +used Ubuntu 20.04 image. + +It can take more than 30 minutes for instantiating a profile provided to +[start-profile.py](start-profile.py) to complete, but you'll see intermittent messages on +`stdout` indicating progress and logging results. In some cases, the Powder +resources required by [start-profile.py](start-profile.py) might not be available. If so, +you'll see a log message that indicates as much and the script will exit; you +can look at the [Resource Availability](https://www.powderwireless.net/resinfo.php) +page on the Powder portal to see the status of the required instance. After +completion, the script will exit with a message about failure/success. diff --git a/XTesting/powder-control/powder/experiment.py b/XTesting/powder-control/powder/experiment.py new file mode 100644 index 0000000..2aa8f9d --- /dev/null +++ b/XTesting/powder-control/powder/experiment.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import json +import logging +import sys +import time + +import xmltodict + +import powder.rpc as prpc +import powder.ssh as pssh + + +class PowderExperiment: + """Represents a single powder experiment. Can be used to start, interact with, + and terminate the experiment. After an experiment is ready, this object + holds references to the nodes in the experiment, which can be interacted + with via ssh. + + Args: + experiment_name (str): A name for the experiment. Must be less than 16 characters. + project_name (str): The name of the Powder Project associated with the experiment. + profile_name (str): The name of an existing Powder profile you want to use for the experiment. + + Attributes: + status (int): Represents the last known status of the experiment as + retrieved from the Powder RPC servqer. + nodes (dict of str: Node): A lookup table mapping node ids to Node instances + in the experiment. + experiment_name (str) + project_name (str) + profile_name (str) + + """ + + EXPERIMENT_NOT_STARTED = 0 + EXPERIMENT_PROVISIONING = 1 + EXPERIMENT_PROVISIONED = 2 + EXPERIMENT_READY = 3 + EXPERIMENT_FAILED = 4 + EXPERIMENT_NULL = 5 + + POLL_INTERVAL_S = 20 + PROVISION_TIMEOUT_S = 1800 + MAX_NAME_LENGTH = 16 + + def __init__(self, experiment_name, project_name, profile_name): + if len(experiment_name) > 16: + logging.error('Experiment name {} is too long (cannot exceed {} characters)'.format(experiment_name, + self.MAX_NAME_LENGTH)) + sys.exit(1) + + self.experiment_name = experiment_name + self.project_name = project_name + self.profile_name = profile_name + self.status = self.EXPERIMENT_NOT_STARTED + self.nodes = dict() + self._manifests = None + self._poll_count_max = self.PROVISION_TIMEOUT_S // self.POLL_INTERVAL_S + logging.info('initialized experiment {} based on profile {} under project {}'.format(experiment_name, + profile_name, + project_name)) + + def start_and_wait(self): + """Start the experiment and wait for READY or FAILED status.""" + logging.info('starting experiment {}'.format(self.experiment_name)) + rval, response = prpc.start_experiment(self.experiment_name, + self.project_name, + self.profile_name) + if rval == prpc.RESPONSE_SUCCESS: + self._get_status() + + poll_count = 0 + logging.info('self.still_provisioning: {}'.format(self.still_provisioning)) + while self.still_provisioning and poll_count < self._poll_count_max: + logging.info('waiting for provision process done') + self._get_status() + time.sleep(self.POLL_INTERVAL_S) + else: + self.status = self.EXPERIMENT_FAILED + logging.info(response) + + return self.status + + def terminate(self): + """Terminate the experiment. All allocated resources will be released.""" + logging.info('terminating experiment {}'.format(self.experiment_name)) + rval, response = prpc.terminate_experiment(self.project_name, self.experiment_name) + if rval == prpc.RESPONSE_SUCCESS: + self.status = self.EXPERIMENT_NULL + else: + logging.error('failed to terminate experiment') + logging.error('output {}'.format(response['output'])) + + return self.status + + def _get_manifests(self): + """Get experiment manifests, translate to list of dicts.""" + rval, response = prpc.get_experiment_manifests(self.project_name, + self.experiment_name) + if rval == prpc.RESPONSE_SUCCESS: + response_json = json.loads(response['output']) + self._manifests = [xmltodict.parse(response_json[key]) for key in response_json.keys()] + logging.info('got manifests') + else: + logging.error('failed to get manifests') + + return self + + def _parse_manifests(self): + """Parse experiment manifests and add nodes to lookup table.""" + for manifest in self._manifests: + logging.info('parsed manifest:{}'.format(manifest)) + nodes = manifest['rspec']['node'] + logging.info('parsed manifest nodes:{}'.format(nodes)) + client_id = nodes['@client_id'] + logging.info('parsed manifest client_id:{}'.format(client_id)) + host = nodes['host'] + logging.info('parsed manifest host:{}'.format(host)) + hostname = host['@name'] + logging.info('parsed manifest hostname:{}'.format(hostname)) + ipv4 = host['@ipv4'] + logging.info('parsed manifest ipv4:{}'.format(ipv4)) + for node in nodes: + logging.info('parsed manifest node:{}'.format(node)) + # only need to add nodes with public IP addresses for now +# try: +# hostname = node['host']['@name'] +# ipv4 = node['host']['@ipv4'] +# client_id = nodes['@client_id'] +# self.nodes[client_id] = Node(client_id=client_id, ip_address=ipv4, +# hostname=hostname) +# except KeyError: +# pass +# + return self + + def _get_status(self): + """Get experiment status and update local state. If the experiment is ready, get + and parse the associated manifests. + + """ + rval, response = prpc.get_experiment_status(self.project_name, + self.experiment_name) + if rval == prpc.RESPONSE_SUCCESS: + output = response['output'] + #if output == 'Status: ready\n': + if "ready" in output: + self.status = self.EXPERIMENT_READY + self._get_manifests()._parse_manifests() + #elif output == 'Status: provisioning\n': + elif "provisioning" in output: + self.status = self.EXPERIMENT_PROVISIONING + #elif output == 'Status: provisioned\n': + elif "provisioned" in output: + self.status = self.EXPERIMENT_PROVISIONED + #elif output == 'Status: failed\n': + elif "failed" in output: + self.status = self.EXPERIMENT_FAILED + + logging.info('status is {}'.format(self.status)) + self.still_provisioning = self.status in [self.EXPERIMENT_PROVISIONING, + self.EXPERIMENT_PROVISIONED] + logging.info('experiment status is {}'.format(output.strip())) + else: + logging.error('failed to get experiment status') + + return self + + +class Node: + """Represents a node on the Powder platform. Holds an SSHConnection instance for + interacting with the node. + + Attributes: + client_id (str): Matches the id defined for the node in the Powder profile. + ip_address (str): The public IP address of the node. + hostname (str): The hostname of the node. + ssh (SSHConnection): For interacting with the node via ssh through pexpect. + + """ + def __init__(self, client_id, ip_address, hostname): + self.client_id = client_id + self.ip_address = ip_address + self.hostname = hostname + self.ssh = pssh.SSHConnection(ip_address=self.ip_address) diff --git a/XTesting/powder-control/powder/rpc.py b/XTesting/powder-control/powder/rpc.py new file mode 100644 index 0000000..5520b7e --- /dev/null +++ b/XTesting/powder-control/powder/rpc.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2004-2020 University of Utah and the Flux Group. +# +# {{{EMULAB-LICENSE +# +# This file is part of the Emulab network testbed software. +# +# This file is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This file is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public +# License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this file. If not, see . +# +# }}} +# + +import logging +import os +import ssl +import sys +import xmlrpc.client as xmlrpc_client + + +PACKAGE_VERSION = 0.1 +DEBUG = 0 + +# rpc server +XMLRPC_SERVER = "boss.emulab.net" +XMLRPC_PORT = 3069 +SERVER_PATH = "/usr/testbed" +URI = "https://" + XMLRPC_SERVER + ":" + str(XMLRPC_PORT) + SERVER_PATH + +RESPONSE_SUCCESS = 0 +RESPONSE_BADARGS = 1 +RESPONSE_ERROR = 2 +RESPONSE_FORBIDDEN = 3 +RESPONSE_BADVERSION = 4 +RESPONSE_SERVERERROR = 5 +RESPONSE_TOOBIG = 6 +RESPONSE_REFUSED = 7 # Emulab is down, try again later. +RESPONSE_TIMEDOUT = 8 + +# User supplied login ID, password, and certificate +try: + LOGIN_ID = os.environ['USER'] + PEM_PWORD = os.environ['PWORD'] + CERT_PATH = os.environ['CERT'] +except KeyError: + logging.error('Missing Powder credential environment variable(s)') + sys.exit(1) + + +def do_method(method, params): + ctx = ssl.SSLContext() + ctx.set_ciphers('ALL:@SECLEVEL=0') + ctx.load_cert_chain(CERT_PATH, password=PEM_PWORD) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + # Get a handle on the server, + server = xmlrpc_client.ServerProxy(URI, context=ctx, verbose=DEBUG) + + # Get a pointer to the function we want to invoke. + meth = getattr(server, "portal." + method) + meth_args = [PACKAGE_VERSION, params] + + # Make the call. + try: + response = meth(*meth_args) + except xmlrpc_client.Fault as e: + print(e.faultString) + return -1, None + + rval = response["code"] + + # If the code indicates failure, look for a "value". Use that as the + # return value instead of the code. + if rval != RESPONSE_SUCCESS: + if response["value"]: + rval = response["value"] + + return rval, response + + +def start_experiment(experiment_name, project_name, profile_name): + params = { + "name": experiment_name, + "proj": project_name, + "profile": ','.join([project_name, profile_name]) + } + rval, response = do_method("startExperiment", params) + return rval, response + + +def terminate_experiment(project_name, experiment_name): + params = { + "experiment": ','.join([project_name, experiment_name]) + } + rval, response = do_method("terminateExperiment", params) + return rval, response + + +def get_experiment_status(project_name, experiment_name): + params = { + "experiment": ','.join([project_name, experiment_name]) + } + rval, response = do_method("experimentStatus", params) + return rval, response + + +def get_experiment_manifests(project_name, experiment_name): + params = { + "experiment": ','.join([project_name, experiment_name]) + } + rval, response = do_method("experimentManifests", params) + return rval, response diff --git a/XTesting/powder-control/requirements.txt b/XTesting/powder-control/requirements.txt new file mode 100644 index 0000000..717679e --- /dev/null +++ b/XTesting/powder-control/requirements.txt @@ -0,0 +1,2 @@ +pexpect==4.6.0 +xmltodict==0.12.0 diff --git a/XTesting/powder-control/start-profile.py b/XTesting/powder-control/start-profile.py new file mode 100755 index 0000000..594dd44 --- /dev/null +++ b/XTesting/powder-control/start-profile.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +import logging +import mmap +import multiprocessing as mp +import powder.experiment as pexp +import random +import re +import string +import sys +import time +import os + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s" +) + + +class PowderProfile: + """Instantiates a Powder experiment based on the provided Powder profile + by default the project is 'osc' and the profile is 'ubuntu-20' + + """ + + # default Powder experiment credentials + PROJECT_NAME = 'osc' + PROFILE_NAME = 'ubuntu-20' + EXPERIMENT_NAME_PREFIX = 'osctest-' + + SUCCEEDED = 0 # all steps succeeded + FAILED = 1 # on of the steps failed + + def __init__(self, experiment_name=None): + if experiment_name is not None: + self.experiment_name = experiment_name + else: + self.experiment_name = self.EXPERIMENT_NAME_PREFIX + self._random_string() + + try: + self.project_name = os.environ['PROJECT'] + except KeyError: + self.project_name = self.PROJECT_NAME + + try: + self.profile_name = os.environ['PROFILE'] + except KeyError: + self.profile_name = self.PROFILE_NAME + + def run(self): + if not self._start_powder_experiment(): + self._finish(self.FAILED) + else: + self._finish(self.SUCCEEDED) + + def _random_string(self, strlen=7): + characters = string.ascii_lowercase + string.digits + return ''.join(random.choice(characters) for i in range(strlen)) + + def _start_powder_experiment(self): + logging.info('Instantiating Powder experiment...') + self.exp = pexp.PowderExperiment(experiment_name=self.experiment_name, + project_name=self.project_name, + profile_name=self.profile_name) + + exp_status = self.exp.start_and_wait() + if exp_status != self.exp.EXPERIMENT_READY: + logging.error('Failed to start experiment.') + return False + else: + return True + + def _finish(self, test_status): + if test_status == self.FAILED: + logging.info('The experiment could not be started... maybe the resources were unavailable.') + elif test_status == self.SUCCEEDED: + logging.info('The experiment successfully started.') + + sys.exit(test_status) + + +if __name__ == '__main__': + xtesting_host = PowderProfile() + xtesting_host.run() -- 2.16.6