add ssh capability and an example on how to execute ssh commands on a newly started...

Change-Id: I7cf05e58e915247d69ce9612981db14b2f515a78
Signed-off-by: pceicicd <pekwatch746@gmail.com>

diff --git a/XTesting/powder-control/powder/experiment.py b/XTesting/powder-control/powder/experiment.py
index 2aa8f9d..cf0e103 100644 (file)
--- a/XTesting/powder-control/powder/experiment.py
+++ b/XTesting/powder-control/powder/experiment.py
@@ -56,6 +56,7 @@ class PowderExperiment:
         self.nodes = dict()
         self._manifests = None
         self._poll_count_max = self.PROVISION_TIMEOUT_S // self.POLL_INTERVAL_S
+        self.ipv4 = None
         logging.info('initialized experiment {} based on profile {} under project {}'.format(experiment_name,
                                                                                              profile_name,
                                                                                              project_name))
@@ -118,8 +119,8 @@ class PowderExperiment:
             logging.info('parsed manifest host:{}'.format(host))
             hostname = host['@name']
             logging.info('parsed manifest hostname:{}'.format(hostname))
-            ipv4 = host['@ipv4']
-            logging.info('parsed manifest ipv4:{}'.format(ipv4))
+            self.ipv4 = host['@ipv4']
+            logging.info('parsed manifest ipv4:{}'.format(self.ipv4))
             for node in nodes:
                 logging.info('parsed manifest node:{}'.format(node))
                 # only need to add nodes with public IP addresses for now
@@ -178,8 +179,9 @@ class Node:
         ssh (SSHConnection): For interacting with the node via ssh through pexpect.
 
     """
-    def __init__(self, client_id, ip_address, hostname):
-        self.client_id = client_id
+    #def __init__(self, client_id, ip_address, hostname):
+    def __init__(self, ip_address):    
+        #self.client_id = client_id
         self.ip_address = ip_address
-        self.hostname = hostname
+        #self.hostname = hostname
         self.ssh = pssh.SSHConnection(ip_address=self.ip_address)

diff --git a/XTesting/powder-control/powder/profile.py b/XTesting/powder-control/powder/profile.py
new file mode 100755 (executable)
index 0000000..eb72237
--- /dev/null
+++ b/XTesting/powder-control/powder/profile.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import logging
+import mmap
+import multiprocessing as mp
+import powder.experiment as pexp
+import random
+import re
+import string
+import sys
+import time
+import os
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s"
+)
+
+
+class PowderProfile:
+    """Instantiates a Powder experiment based on the provided Powder profile 
+    by default the project is 'osc' and the profile is 'ubuntu-20'. 
+    It returns the IPv4 address if a profile starts successfully on Powder
+    """
+
+    # default Powder experiment credentials
+    PROJECT_NAME = 'osc'
+    PROFILE_NAME = 'ubuntu-20'
+    EXPERIMENT_NAME_PREFIX = 'osctest-'
+
+    SUCCEEDED   = 0  # all steps succeeded
+    FAILED      = 1  # on of the steps failed
+
+    def __init__(self, experiment_name=None):
+        if experiment_name is not None:
+            self.experiment_name = experiment_name
+        else:
+            self.experiment_name = self.EXPERIMENT_NAME_PREFIX + self._random_string()
+
+        try:
+            self.project_name = os.environ['PROJECT']
+        except KeyError:
+            self.project_name = self.PROJECT_NAME
+
+        try:
+            self.profile_name = os.environ['PROFILE']
+        except KeyError:
+            self.profile_name = self.PROFILE_NAME
+
+    def run(self):
+        if not self._start_powder_experiment():
+            return self._finish(self.FAILED)
+        else:
+            return self._finish(self.SUCCEEDED)
+
+    def _random_string(self, strlen=7):
+        characters = string.ascii_lowercase + string.digits
+        return ''.join(random.choice(characters) for i in range(strlen))
+
+    def _start_powder_experiment(self):
+        logging.info('Instantiating Powder experiment...')
+        self.exp = pexp.PowderExperiment(experiment_name=self.experiment_name,
+                                         project_name=self.project_name,
+                                         profile_name=self.profile_name)
+
+        exp_status = self.exp.start_and_wait()
+        if exp_status != self.exp.EXPERIMENT_READY:
+            logging.error('Failed to start experiment.')
+            return False
+        else:
+            return True
+
+    def _finish(self, test_status):
+        if test_status == self.FAILED:
+            logging.info('The experiment could not be started... maybe the resources were unavailable.')
+            return test_status, None
+        elif test_status == self.SUCCEEDED:
+            logging.info('The experiment successfully started.')
+            return test_status, self.exp.ipv4

diff --git a/XTesting/powder-control/powder/ssh.py b/XTesting/powder-control/powder/ssh.py
new file mode 100644 (file)
index 0000000..3caec31
--- /dev/null
+++ b/XTesting/powder-control/powder/ssh.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+import logging
+import os
+import pexpect
+import re
+import time
+
+
+class SSHConnection:
+    """A simple ssh/scp wrapper for creating and interacting with ssh sessions via
+    pexpect.
+
+    Args:
+        ip_address (str): IP address of the node.
+        username (str): A username with access to the node.
+        prompt (str) (optional): Expected prompt on the host.
+
+    Attributes:
+        ssh (pexpect child): A handle to a session started by pexpect.spawn()
+    """
+
+    DEFAULT_PROMPT = '\$'
+
+    def __init__(self, ip_address, username=None, password=None, prompt=DEFAULT_PROMPT, key=None):
+        self.prompt = prompt
+        self.ip_address = ip_address
+        self.key = key
+        if username is None:
+            try:
+                self.username = os.environ['USER']
+            except KeyError:
+                logging.error('no USER variable in environment variable and no username provided')
+                raise ValueError
+
+        if password is None:
+            try:
+                self.password = os.environ['KEYPWORD']
+            except KeyError:
+                logging.info('no ssh key password in environment, assuming unencrypted')
+                self.password = password
+
+        if key is None:
+            try:
+                self.key = os.environ['CERT']
+            except KeyError:
+                logging.info('no ssh key path provided in environment, assuming not using a ssh key')
+                self.key = key
+
+    def open(self):
+        """Opens a connection to `self.ip_address` using `self.username`."""
+
+        retry_count = 0
+        if self.key is None:
+            cmd = 'ssh -l {} {}'.format(self.username, self.ip_address)
+        else:
+            # this is to provide a public key for the ssh session, which might be the most
+            # commonly use case
+            cmd = 'ssh -i {} -l {} {}'.format(self.key, self.username, self.ip_address)
+        while retry_count < 4:
+            self.ssh = pexpect.spawn(cmd, timeout=5)
+            self.sshresponse = self.ssh.expect([self.prompt,
+                                                'Are you sure you want to continue connecting (yes/no)?',
+                                                'Last login',
+                                                'Enter passphrase for key.*:',
+                                                pexpect.EOF,
+                                                pexpect.TIMEOUT])
+            if self.sshresponse == 0:
+                return self
+            elif self.sshresponse == 1:
+                self.ssh.sendline('yes')
+                self.sshresponse = self.ssh.expect([self.prompt,
+                                                    'Enter passphrase for key.*:',
+                                                    'Permission denied',
+                                                    pexpect.EOF,
+                                                    pexpect.TIMEOUT])
+                if self.sshresponse == 0:
+                    return self
+                elif self.sshresponse == 1:
+                    if self.password is None:
+                        logging.error('failed to login --- ssh key is encrypted but no pword provided')
+                        raise ValueError
+
+                    self.ssh.sendline(self.password)
+                    self.sshresponse = self.ssh.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+                    if self.sshresponse == 0:
+                        return self
+                    else:
+                        logging.debug('failed to login --- response: {}'.format(self.sshresponse))
+                        logging.debug('retry count: {}'.format(retry_count))
+                else:
+                    logging.debug('failed to login --- response: {}'.format(self.sshresponse))
+                    logging.debug('retry count: {}'.format(retry_count))
+
+            elif self.sshresponse == 2:
+                # Verify we've connected to the self.ip_address
+                self.command('ifconfig | egrep --color=never "inet addr:|inet "', self.prompt)
+                self.sshresponse = self.ssh.expect([self.prompt, pexpect.EOF, pexpect.TIMEOUT])
+                result = re.search(str(self.ip_address), str(self.ssh.before))
+                if result is None:
+                    logging.debug('not on host with ip {}'.format(self.ip_address))
+                    logging.debug('retry count: {}'.format(retry_count))
+                else:
+                    return self
+
+            elif self.sshresponse == 3:
+                if self.password is None:
+                    logging.error('failed to login --- ssh key is encrypted but no pword provided')
+                    raise ValueError
+
+                self.ssh.sendline(self.password)
+                self.sshresponse = self.ssh.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+
+                if self.sshresponse == 0:
+                    return self
+                else:
+                    logging.debug('failed to login --- response: {}'.format(self.sshresponse))
+                    logging.debug('retry count: {}'.format(retry_count))
+
+            elif self.sshresponse == 4:
+                logging.debug('Unexpected EOF')
+                logging.debug('retry count: {}'.format(retry_count))
+                logging.debug('ssh.before: ' + str(self.ssh.before))
+            elif self.sshresponse == 5:
+                logging.debug('Unexpected Timeout')
+                logging.debug('retry count: {}'.format(retry_count))
+                logging.debug('ssh.before: ' + str(self.ssh.before))
+
+            time.sleep(1)
+            retry_count += 1
+
+        logging.error('failed to login --- could not connect to host.')
+        raise ValueError
+
+    def command(self, commandline, expectedline=DEFAULT_PROMPT, timeout=5):
+        """Sends `commandline` to `self.ip_address` and waits for `expectedline`."""
+        logging.debug(commandline)
+        self.ssh.sendline(commandline)
+        self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT], timeout=timeout)
+        if self.sshresponse == 0:
+            pass
+        elif self.sshresponse == 1:
+            logging.debug('Unexpected EOF --- Expected: ' + expectedline)
+            logging.debug('ssh.before: ' + str(self.ssh.before))
+        elif self.sshresponse == 2:
+            logging.debug('Unexpected Timeout --- Expected: ' + expectedline)
+            logging.debug('ssh.before: ' + str(self.ssh.before))
+
+        return self.sshresponse
+
+    def close(self, timeout):
+        self.ssh.sendline('exit')
+        self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=timeout)
+        if self.sshresponse == 0:
+            pass
+        elif self.sshresponse == 1:
+            logging.debug('Unexpected Timeout --- Expected: EOF')
+            logging.debug('ssh.before: ' + str(self.ssh.before))
+
+        return self.sshresponse
+
+    def copy_from(self, remote_path, local_path):
+        cmd = 'scp {}@{}:{} {}'.format(self.username, self.ip_address, remote_path, local_path)
+        return self.copy(cmd)
+
+    def copy_to(self, local_path, remote_path):
+        cmd = 'scp {} {}@{}:{}'.format(local_path, self.username, self.ip_address, remote_path)
+        return self.copy(cmd)
+
+    def copy(self, cmd):
+        retry_count = 0
+        logging.debug(cmd)
+        while retry_count < 10:
+            scp_spawn = pexpect.spawn(cmd, timeout = 100)
+            scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?',
+                                             'Enter passphrase for key.*:',
+                                             pexpect.EOF,
+                                             pexpect.TIMEOUT])
+            if scp_response == 0:
+                scp_spawn.sendline('yes')
+                scp_response = scp_spawn.expect([self.prompt,
+                                                 'Enter passphrase for key.*:',
+                                                 'Permission denied',
+                                                 pexpect.EOF,
+                                                 pexpect.TIMEOUT])
+                if scp_response == 0:
+                    return scp_response
+                elif scp_response == 1:
+                    if self.password is None:
+                        logging.error('failed to scp --- ssh key is encrypted but no pword provided')
+                        raise ValueError
+
+                    scp_spawn.sendline(self.password)
+                    scp_response = scp_spawn.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+                    if scp_response == 0:
+                        return scp_response
+                    else:
+                        logging.debug('failed to scp --- response: {}'.format(scp_response))
+                        logging.debug('retry count: {}'.format(retry_count))
+                    return scp_response
+                else:
+                    logging.debug('scp failed with scp response {}'.format(scp_response))
+                    logging.debug('retry count: {}'.format(retry_count))
+            elif scp_response == 1:
+                if self.password is None:
+                    logging.error('failed to scp --- ssh key is encrypted but no pword provided')
+                    raise ValueError
+
+                scp_spawn.sendline(self.password)
+                scp_response = scp_spawn.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+                if scp_response == 0:
+                    return scp_response
+                else:
+                    logging.debug('failed to scp --- response: {}'.format(scp_response))
+                    logging.debug('retry count: {}'.format(retry_count))
+                return scp_response
+            elif scp_response == 2:
+                logging.debug('copy succeeded')
+                return scp_response
+
+            time.sleep(1)
+            retry_count += 1
+
+        return scp_response

diff --git a/XTesting/powder-control/run_profile_example.py b/XTesting/powder-control/run_profile_example.py
new file mode 100755 (executable)
index 0000000..33717a4
--- /dev/null
+++ b/XTesting/powder-control/run_profile_example.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import logging
+import mmap
+import multiprocessing as mp
+import powder.experiment as pexp
+import random
+import re
+import string
+import sys
+import time
+from powder.profile import PowderProfile
+
+class RunTest:
+    """Based on the return values from the process that starts a given POWDER profile,
+    create ssh connection and run desired commands, e.g., test steps that's already
+    published as scripts or pure commands
+
+    """
+
+    TEST_SUCCEEDED   = 0  # all steps succeeded
+    TEST_FAILED      = 1  # one of the steps failed
+    TEST_NOT_STARTED = 2  # could not instantiate an experiment to run the test on
+
+    def run(self):
+        powder_host = PowderProfile()
+        # expect the start on a specified profile succeeds and return
+        # an IPv4 address to proceed to the next step
+        status, ip_address = powder_host.run()
+
+        if not ip_address:
+            sys.exit(self.TEST_NOT_STARTED)
+        elif self._start_powder_experiment(ip_address):
+            sys.exit(self.TEST_FAILED)
+        else:    
+            sys.exit(self.TEST_SUCCEEDED)
+
+    def _start_powder_experiment(self, ip_address):
+        logging.info('Executing ssh commands on host:{}'.format(ip_address))
+        node = pexp.Node(ip_address=ip_address)
+        ssh_node = node.ssh.open()
+        # the example commands shown below are to set up the AI/ML FW from scratch
+        ssh_node.command('sudo groupadd docker && sudo usermod -aG docker osc_int')
+        ssh_node.close(5)
+        ssh_node = node.ssh.open()
+        ssh_node.command('git clone https://gerrit.o-ran-sc.org/r/aiml-fw/aimlfw-dep')
+        ssh_node.command('cd aimlfw-dep && bin/install_traininghost.sh 2>&1 | tee /tmp/install.log', timeout=1800)
+        ssh_node.close(5)
+
+if __name__ == '__main__':
+    powdertest = RunTest()
+    powdertest.run()