From 73ae93dedb80349ba65a066af8c2ed5d3e4d1323 Mon Sep 17 00:00:00 2001 From: pceicicd Date: Wed, 18 Oct 2023 19:05:12 -0700 Subject: [PATCH] add ssh capability and an example on how to execute ssh commands on a newly started experiment Change-Id: I7cf05e58e915247d69ce9612981db14b2f515a78 Signed-off-by: pceicicd --- XTesting/powder-control/powder/experiment.py | 12 +- XTesting/powder-control/powder/profile.py | 78 +++++++++ XTesting/powder-control/powder/ssh.py | 223 +++++++++++++++++++++++++ XTesting/powder-control/run_profile_example.py | 51 ++++++ 4 files changed, 359 insertions(+), 5 deletions(-) create mode 100755 XTesting/powder-control/powder/profile.py create mode 100644 XTesting/powder-control/powder/ssh.py create mode 100755 XTesting/powder-control/run_profile_example.py diff --git a/XTesting/powder-control/powder/experiment.py b/XTesting/powder-control/powder/experiment.py index 2aa8f9d..cf0e103 100644 --- a/XTesting/powder-control/powder/experiment.py +++ b/XTesting/powder-control/powder/experiment.py @@ -56,6 +56,7 @@ class PowderExperiment: self.nodes = dict() self._manifests = None self._poll_count_max = self.PROVISION_TIMEOUT_S // self.POLL_INTERVAL_S + self.ipv4 = None logging.info('initialized experiment {} based on profile {} under project {}'.format(experiment_name, profile_name, project_name)) @@ -118,8 +119,8 @@ class PowderExperiment: logging.info('parsed manifest host:{}'.format(host)) hostname = host['@name'] logging.info('parsed manifest hostname:{}'.format(hostname)) - ipv4 = host['@ipv4'] - logging.info('parsed manifest ipv4:{}'.format(ipv4)) + self.ipv4 = host['@ipv4'] + logging.info('parsed manifest ipv4:{}'.format(self.ipv4)) for node in nodes: logging.info('parsed manifest node:{}'.format(node)) # only need to add nodes with public IP addresses for now @@ -178,8 +179,9 @@ class Node: ssh (SSHConnection): For interacting with the node via ssh through pexpect. """ - def __init__(self, client_id, ip_address, hostname): - self.client_id = client_id + #def __init__(self, client_id, ip_address, hostname): + def __init__(self, ip_address): + #self.client_id = client_id self.ip_address = ip_address - self.hostname = hostname + #self.hostname = hostname self.ssh = pssh.SSHConnection(ip_address=self.ip_address) diff --git a/XTesting/powder-control/powder/profile.py b/XTesting/powder-control/powder/profile.py new file mode 100755 index 0000000..eb72237 --- /dev/null +++ b/XTesting/powder-control/powder/profile.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import logging +import mmap +import multiprocessing as mp +import powder.experiment as pexp +import random +import re +import string +import sys +import time +import os + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s" +) + + +class PowderProfile: + """Instantiates a Powder experiment based on the provided Powder profile + by default the project is 'osc' and the profile is 'ubuntu-20'. + It returns the IPv4 address if a profile starts successfully on Powder + """ + + # default Powder experiment credentials + PROJECT_NAME = 'osc' + PROFILE_NAME = 'ubuntu-20' + EXPERIMENT_NAME_PREFIX = 'osctest-' + + SUCCEEDED = 0 # all steps succeeded + FAILED = 1 # on of the steps failed + + def __init__(self, experiment_name=None): + if experiment_name is not None: + self.experiment_name = experiment_name + else: + self.experiment_name = self.EXPERIMENT_NAME_PREFIX + self._random_string() + + try: + self.project_name = os.environ['PROJECT'] + except KeyError: + self.project_name = self.PROJECT_NAME + + try: + self.profile_name = os.environ['PROFILE'] + except KeyError: + self.profile_name = self.PROFILE_NAME + + def run(self): + if not self._start_powder_experiment(): + return self._finish(self.FAILED) + else: + return self._finish(self.SUCCEEDED) + + def _random_string(self, strlen=7): + characters = string.ascii_lowercase + string.digits + return ''.join(random.choice(characters) for i in range(strlen)) + + def _start_powder_experiment(self): + logging.info('Instantiating Powder experiment...') + self.exp = pexp.PowderExperiment(experiment_name=self.experiment_name, + project_name=self.project_name, + profile_name=self.profile_name) + + exp_status = self.exp.start_and_wait() + if exp_status != self.exp.EXPERIMENT_READY: + logging.error('Failed to start experiment.') + return False + else: + return True + + def _finish(self, test_status): + if test_status == self.FAILED: + logging.info('The experiment could not be started... maybe the resources were unavailable.') + return test_status, None + elif test_status == self.SUCCEEDED: + logging.info('The experiment successfully started.') + return test_status, self.exp.ipv4 diff --git a/XTesting/powder-control/powder/ssh.py b/XTesting/powder-control/powder/ssh.py new file mode 100644 index 0000000..3caec31 --- /dev/null +++ b/XTesting/powder-control/powder/ssh.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +import logging +import os +import pexpect +import re +import time + + +class SSHConnection: + """A simple ssh/scp wrapper for creating and interacting with ssh sessions via + pexpect. + + Args: + ip_address (str): IP address of the node. + username (str): A username with access to the node. + prompt (str) (optional): Expected prompt on the host. + + Attributes: + ssh (pexpect child): A handle to a session started by pexpect.spawn() + """ + + DEFAULT_PROMPT = '\$' + + def __init__(self, ip_address, username=None, password=None, prompt=DEFAULT_PROMPT, key=None): + self.prompt = prompt + self.ip_address = ip_address + self.key = key + if username is None: + try: + self.username = os.environ['USER'] + except KeyError: + logging.error('no USER variable in environment variable and no username provided') + raise ValueError + + if password is None: + try: + self.password = os.environ['KEYPWORD'] + except KeyError: + logging.info('no ssh key password in environment, assuming unencrypted') + self.password = password + + if key is None: + try: + self.key = os.environ['CERT'] + except KeyError: + logging.info('no ssh key path provided in environment, assuming not using a ssh key') + self.key = key + + def open(self): + """Opens a connection to `self.ip_address` using `self.username`.""" + + retry_count = 0 + if self.key is None: + cmd = 'ssh -l {} {}'.format(self.username, self.ip_address) + else: + # this is to provide a public key for the ssh session, which might be the most + # commonly use case + cmd = 'ssh -i {} -l {} {}'.format(self.key, self.username, self.ip_address) + while retry_count < 4: + self.ssh = pexpect.spawn(cmd, timeout=5) + self.sshresponse = self.ssh.expect([self.prompt, + 'Are you sure you want to continue connecting (yes/no)?', + 'Last login', + 'Enter passphrase for key.*:', + pexpect.EOF, + pexpect.TIMEOUT]) + if self.sshresponse == 0: + return self + elif self.sshresponse == 1: + self.ssh.sendline('yes') + self.sshresponse = self.ssh.expect([self.prompt, + 'Enter passphrase for key.*:', + 'Permission denied', + pexpect.EOF, + pexpect.TIMEOUT]) + if self.sshresponse == 0: + return self + elif self.sshresponse == 1: + if self.password is None: + logging.error('failed to login --- ssh key is encrypted but no pword provided') + raise ValueError + + self.ssh.sendline(self.password) + self.sshresponse = self.ssh.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT]) + if self.sshresponse == 0: + return self + else: + logging.debug('failed to login --- response: {}'.format(self.sshresponse)) + logging.debug('retry count: {}'.format(retry_count)) + else: + logging.debug('failed to login --- response: {}'.format(self.sshresponse)) + logging.debug('retry count: {}'.format(retry_count)) + + elif self.sshresponse == 2: + # Verify we've connected to the self.ip_address + self.command('ifconfig | egrep --color=never "inet addr:|inet "', self.prompt) + self.sshresponse = self.ssh.expect([self.prompt, pexpect.EOF, pexpect.TIMEOUT]) + result = re.search(str(self.ip_address), str(self.ssh.before)) + if result is None: + logging.debug('not on host with ip {}'.format(self.ip_address)) + logging.debug('retry count: {}'.format(retry_count)) + else: + return self + + elif self.sshresponse == 3: + if self.password is None: + logging.error('failed to login --- ssh key is encrypted but no pword provided') + raise ValueError + + self.ssh.sendline(self.password) + self.sshresponse = self.ssh.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT]) + + if self.sshresponse == 0: + return self + else: + logging.debug('failed to login --- response: {}'.format(self.sshresponse)) + logging.debug('retry count: {}'.format(retry_count)) + + elif self.sshresponse == 4: + logging.debug('Unexpected EOF') + logging.debug('retry count: {}'.format(retry_count)) + logging.debug('ssh.before: ' + str(self.ssh.before)) + elif self.sshresponse == 5: + logging.debug('Unexpected Timeout') + logging.debug('retry count: {}'.format(retry_count)) + logging.debug('ssh.before: ' + str(self.ssh.before)) + + time.sleep(1) + retry_count += 1 + + logging.error('failed to login --- could not connect to host.') + raise ValueError + + def command(self, commandline, expectedline=DEFAULT_PROMPT, timeout=5): + """Sends `commandline` to `self.ip_address` and waits for `expectedline`.""" + logging.debug(commandline) + self.ssh.sendline(commandline) + self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT], timeout=timeout) + if self.sshresponse == 0: + pass + elif self.sshresponse == 1: + logging.debug('Unexpected EOF --- Expected: ' + expectedline) + logging.debug('ssh.before: ' + str(self.ssh.before)) + elif self.sshresponse == 2: + logging.debug('Unexpected Timeout --- Expected: ' + expectedline) + logging.debug('ssh.before: ' + str(self.ssh.before)) + + return self.sshresponse + + def close(self, timeout): + self.ssh.sendline('exit') + self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=timeout) + if self.sshresponse == 0: + pass + elif self.sshresponse == 1: + logging.debug('Unexpected Timeout --- Expected: EOF') + logging.debug('ssh.before: ' + str(self.ssh.before)) + + return self.sshresponse + + def copy_from(self, remote_path, local_path): + cmd = 'scp {}@{}:{} {}'.format(self.username, self.ip_address, remote_path, local_path) + return self.copy(cmd) + + def copy_to(self, local_path, remote_path): + cmd = 'scp {} {}@{}:{}'.format(local_path, self.username, self.ip_address, remote_path) + return self.copy(cmd) + + def copy(self, cmd): + retry_count = 0 + logging.debug(cmd) + while retry_count < 10: + scp_spawn = pexpect.spawn(cmd, timeout = 100) + scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', + 'Enter passphrase for key.*:', + pexpect.EOF, + pexpect.TIMEOUT]) + if scp_response == 0: + scp_spawn.sendline('yes') + scp_response = scp_spawn.expect([self.prompt, + 'Enter passphrase for key.*:', + 'Permission denied', + pexpect.EOF, + pexpect.TIMEOUT]) + if scp_response == 0: + return scp_response + elif scp_response == 1: + if self.password is None: + logging.error('failed to scp --- ssh key is encrypted but no pword provided') + raise ValueError + + scp_spawn.sendline(self.password) + scp_response = scp_spawn.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT]) + if scp_response == 0: + return scp_response + else: + logging.debug('failed to scp --- response: {}'.format(scp_response)) + logging.debug('retry count: {}'.format(retry_count)) + return scp_response + else: + logging.debug('scp failed with scp response {}'.format(scp_response)) + logging.debug('retry count: {}'.format(retry_count)) + elif scp_response == 1: + if self.password is None: + logging.error('failed to scp --- ssh key is encrypted but no pword provided') + raise ValueError + + scp_spawn.sendline(self.password) + scp_response = scp_spawn.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT]) + if scp_response == 0: + return scp_response + else: + logging.debug('failed to scp --- response: {}'.format(scp_response)) + logging.debug('retry count: {}'.format(retry_count)) + return scp_response + elif scp_response == 2: + logging.debug('copy succeeded') + return scp_response + + time.sleep(1) + retry_count += 1 + + return scp_response diff --git a/XTesting/powder-control/run_profile_example.py b/XTesting/powder-control/run_profile_example.py new file mode 100755 index 0000000..33717a4 --- /dev/null +++ b/XTesting/powder-control/run_profile_example.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +import logging +import mmap +import multiprocessing as mp +import powder.experiment as pexp +import random +import re +import string +import sys +import time +from powder.profile import PowderProfile + +class RunTest: + """Based on the return values from the process that starts a given POWDER profile, + create ssh connection and run desired commands, e.g., test steps that's already + published as scripts or pure commands + + """ + + TEST_SUCCEEDED = 0 # all steps succeeded + TEST_FAILED = 1 # one of the steps failed + TEST_NOT_STARTED = 2 # could not instantiate an experiment to run the test on + + def run(self): + powder_host = PowderProfile() + # expect the start on a specified profile succeeds and return + # an IPv4 address to proceed to the next step + status, ip_address = powder_host.run() + + if not ip_address: + sys.exit(self.TEST_NOT_STARTED) + elif self._start_powder_experiment(ip_address): + sys.exit(self.TEST_FAILED) + else: + sys.exit(self.TEST_SUCCEEDED) + + def _start_powder_experiment(self, ip_address): + logging.info('Executing ssh commands on host:{}'.format(ip_address)) + node = pexp.Node(ip_address=ip_address) + ssh_node = node.ssh.open() + # the example commands shown below are to set up the AI/ML FW from scratch + ssh_node.command('sudo groupadd docker && sudo usermod -aG docker osc_int') + ssh_node.close(5) + ssh_node = node.ssh.open() + ssh_node.command('git clone https://gerrit.o-ran-sc.org/r/aiml-fw/aimlfw-dep') + ssh_node.command('cd aimlfw-dep && bin/install_traininghost.sh 2>&1 | tee /tmp/install.log', timeout=1800) + ssh_node.close(5) + +if __name__ == '__main__': + powdertest = RunTest() + powdertest.run() -- 2.16.6