--- /dev/null
+#!/usr/bin/env python3
+import logging
+import mmap
+import multiprocessing as mp
+import powder.experiment as pexp
+import random
+import re
+import string
+import sys
+import time
+import os
+
+logging.basicConfig(
+ level=logging.DEBUG,
+ format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s"
+)
+
+
+class PowderProfile:
+ """Instantiates a Powder experiment based on the provided Powder profile
+ by default the project is 'osc' and the profile is 'ubuntu-20'.
+ It returns the IPv4 address if a profile starts successfully on Powder
+ """
+
+ # default Powder experiment credentials
+ PROJECT_NAME = 'osc'
+ PROFILE_NAME = 'ubuntu-20'
+ EXPERIMENT_NAME_PREFIX = 'osctest-'
+
+ SUCCEEDED = 0 # all steps succeeded
+ FAILED = 1 # on of the steps failed
+
+ def __init__(self, experiment_name=None):
+ if experiment_name is not None:
+ self.experiment_name = experiment_name
+ else:
+ self.experiment_name = self.EXPERIMENT_NAME_PREFIX + self._random_string()
+
+ try:
+ self.project_name = os.environ['PROJECT']
+ except KeyError:
+ self.project_name = self.PROJECT_NAME
+
+ try:
+ self.profile_name = os.environ['PROFILE']
+ except KeyError:
+ self.profile_name = self.PROFILE_NAME
+
+ def run(self):
+ if not self._start_powder_experiment():
+ return self._finish(self.FAILED)
+ else:
+ return self._finish(self.SUCCEEDED)
+
+ def _random_string(self, strlen=7):
+ characters = string.ascii_lowercase + string.digits
+ return ''.join(random.choice(characters) for i in range(strlen))
+
+ def _start_powder_experiment(self):
+ logging.info('Instantiating Powder experiment...')
+ self.exp = pexp.PowderExperiment(experiment_name=self.experiment_name,
+ project_name=self.project_name,
+ profile_name=self.profile_name)
+
+ exp_status = self.exp.start_and_wait()
+ if exp_status != self.exp.EXPERIMENT_READY:
+ logging.error('Failed to start experiment.')
+ return False
+ else:
+ return True
+
+ def _finish(self, test_status):
+ if test_status == self.FAILED:
+ logging.info('The experiment could not be started... maybe the resources were unavailable.')
+ return test_status, None
+ elif test_status == self.SUCCEEDED:
+ logging.info('The experiment successfully started.')
+ return test_status, self.exp.ipv4
--- /dev/null
+#!/usr/bin/env python3
+import logging
+import os
+import pexpect
+import re
+import time
+
+
+class SSHConnection:
+ """A simple ssh/scp wrapper for creating and interacting with ssh sessions via
+ pexpect.
+
+ Args:
+ ip_address (str): IP address of the node.
+ username (str): A username with access to the node.
+ prompt (str) (optional): Expected prompt on the host.
+
+ Attributes:
+ ssh (pexpect child): A handle to a session started by pexpect.spawn()
+ """
+
+ DEFAULT_PROMPT = '\$'
+
+ def __init__(self, ip_address, username=None, password=None, prompt=DEFAULT_PROMPT, key=None):
+ self.prompt = prompt
+ self.ip_address = ip_address
+ self.key = key
+ if username is None:
+ try:
+ self.username = os.environ['USER']
+ except KeyError:
+ logging.error('no USER variable in environment variable and no username provided')
+ raise ValueError
+
+ if password is None:
+ try:
+ self.password = os.environ['KEYPWORD']
+ except KeyError:
+ logging.info('no ssh key password in environment, assuming unencrypted')
+ self.password = password
+
+ if key is None:
+ try:
+ self.key = os.environ['CERT']
+ except KeyError:
+ logging.info('no ssh key path provided in environment, assuming not using a ssh key')
+ self.key = key
+
+ def open(self):
+ """Opens a connection to `self.ip_address` using `self.username`."""
+
+ retry_count = 0
+ if self.key is None:
+ cmd = 'ssh -l {} {}'.format(self.username, self.ip_address)
+ else:
+ # this is to provide a public key for the ssh session, which might be the most
+ # commonly use case
+ cmd = 'ssh -i {} -l {} {}'.format(self.key, self.username, self.ip_address)
+ while retry_count < 4:
+ self.ssh = pexpect.spawn(cmd, timeout=5)
+ self.sshresponse = self.ssh.expect([self.prompt,
+ 'Are you sure you want to continue connecting (yes/no)?',
+ 'Last login',
+ 'Enter passphrase for key.*:',
+ pexpect.EOF,
+ pexpect.TIMEOUT])
+ if self.sshresponse == 0:
+ return self
+ elif self.sshresponse == 1:
+ self.ssh.sendline('yes')
+ self.sshresponse = self.ssh.expect([self.prompt,
+ 'Enter passphrase for key.*:',
+ 'Permission denied',
+ pexpect.EOF,
+ pexpect.TIMEOUT])
+ if self.sshresponse == 0:
+ return self
+ elif self.sshresponse == 1:
+ if self.password is None:
+ logging.error('failed to login --- ssh key is encrypted but no pword provided')
+ raise ValueError
+
+ self.ssh.sendline(self.password)
+ self.sshresponse = self.ssh.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+ if self.sshresponse == 0:
+ return self
+ else:
+ logging.debug('failed to login --- response: {}'.format(self.sshresponse))
+ logging.debug('retry count: {}'.format(retry_count))
+ else:
+ logging.debug('failed to login --- response: {}'.format(self.sshresponse))
+ logging.debug('retry count: {}'.format(retry_count))
+
+ elif self.sshresponse == 2:
+ # Verify we've connected to the self.ip_address
+ self.command('ifconfig | egrep --color=never "inet addr:|inet "', self.prompt)
+ self.sshresponse = self.ssh.expect([self.prompt, pexpect.EOF, pexpect.TIMEOUT])
+ result = re.search(str(self.ip_address), str(self.ssh.before))
+ if result is None:
+ logging.debug('not on host with ip {}'.format(self.ip_address))
+ logging.debug('retry count: {}'.format(retry_count))
+ else:
+ return self
+
+ elif self.sshresponse == 3:
+ if self.password is None:
+ logging.error('failed to login --- ssh key is encrypted but no pword provided')
+ raise ValueError
+
+ self.ssh.sendline(self.password)
+ self.sshresponse = self.ssh.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+
+ if self.sshresponse == 0:
+ return self
+ else:
+ logging.debug('failed to login --- response: {}'.format(self.sshresponse))
+ logging.debug('retry count: {}'.format(retry_count))
+
+ elif self.sshresponse == 4:
+ logging.debug('Unexpected EOF')
+ logging.debug('retry count: {}'.format(retry_count))
+ logging.debug('ssh.before: ' + str(self.ssh.before))
+ elif self.sshresponse == 5:
+ logging.debug('Unexpected Timeout')
+ logging.debug('retry count: {}'.format(retry_count))
+ logging.debug('ssh.before: ' + str(self.ssh.before))
+
+ time.sleep(1)
+ retry_count += 1
+
+ logging.error('failed to login --- could not connect to host.')
+ raise ValueError
+
+ def command(self, commandline, expectedline=DEFAULT_PROMPT, timeout=5):
+ """Sends `commandline` to `self.ip_address` and waits for `expectedline`."""
+ logging.debug(commandline)
+ self.ssh.sendline(commandline)
+ self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT], timeout=timeout)
+ if self.sshresponse == 0:
+ pass
+ elif self.sshresponse == 1:
+ logging.debug('Unexpected EOF --- Expected: ' + expectedline)
+ logging.debug('ssh.before: ' + str(self.ssh.before))
+ elif self.sshresponse == 2:
+ logging.debug('Unexpected Timeout --- Expected: ' + expectedline)
+ logging.debug('ssh.before: ' + str(self.ssh.before))
+
+ return self.sshresponse
+
+ def close(self, timeout):
+ self.ssh.sendline('exit')
+ self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=timeout)
+ if self.sshresponse == 0:
+ pass
+ elif self.sshresponse == 1:
+ logging.debug('Unexpected Timeout --- Expected: EOF')
+ logging.debug('ssh.before: ' + str(self.ssh.before))
+
+ return self.sshresponse
+
+ def copy_from(self, remote_path, local_path):
+ cmd = 'scp {}@{}:{} {}'.format(self.username, self.ip_address, remote_path, local_path)
+ return self.copy(cmd)
+
+ def copy_to(self, local_path, remote_path):
+ cmd = 'scp {} {}@{}:{}'.format(local_path, self.username, self.ip_address, remote_path)
+ return self.copy(cmd)
+
+ def copy(self, cmd):
+ retry_count = 0
+ logging.debug(cmd)
+ while retry_count < 10:
+ scp_spawn = pexpect.spawn(cmd, timeout = 100)
+ scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?',
+ 'Enter passphrase for key.*:',
+ pexpect.EOF,
+ pexpect.TIMEOUT])
+ if scp_response == 0:
+ scp_spawn.sendline('yes')
+ scp_response = scp_spawn.expect([self.prompt,
+ 'Enter passphrase for key.*:',
+ 'Permission denied',
+ pexpect.EOF,
+ pexpect.TIMEOUT])
+ if scp_response == 0:
+ return scp_response
+ elif scp_response == 1:
+ if self.password is None:
+ logging.error('failed to scp --- ssh key is encrypted but no pword provided')
+ raise ValueError
+
+ scp_spawn.sendline(self.password)
+ scp_response = scp_spawn.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+ if scp_response == 0:
+ return scp_response
+ else:
+ logging.debug('failed to scp --- response: {}'.format(scp_response))
+ logging.debug('retry count: {}'.format(retry_count))
+ return scp_response
+ else:
+ logging.debug('scp failed with scp response {}'.format(scp_response))
+ logging.debug('retry count: {}'.format(retry_count))
+ elif scp_response == 1:
+ if self.password is None:
+ logging.error('failed to scp --- ssh key is encrypted but no pword provided')
+ raise ValueError
+
+ scp_spawn.sendline(self.password)
+ scp_response = scp_spawn.expect([self.prompt, 'Permission denied', pexpect.EOF, pexpect.TIMEOUT])
+ if scp_response == 0:
+ return scp_response
+ else:
+ logging.debug('failed to scp --- response: {}'.format(scp_response))
+ logging.debug('retry count: {}'.format(retry_count))
+ return scp_response
+ elif scp_response == 2:
+ logging.debug('copy succeeded')
+ return scp_response
+
+ time.sleep(1)
+ retry_count += 1
+
+ return scp_response
--- /dev/null
+#!/usr/bin/env python3
+import logging
+import mmap
+import multiprocessing as mp
+import powder.experiment as pexp
+import random
+import re
+import string
+import sys
+import time
+from powder.profile import PowderProfile
+
+class RunTest:
+ """Based on the return values from the process that starts a given POWDER profile,
+ create ssh connection and run desired commands, e.g., test steps that's already
+ published as scripts or pure commands
+
+ """
+
+ TEST_SUCCEEDED = 0 # all steps succeeded
+ TEST_FAILED = 1 # one of the steps failed
+ TEST_NOT_STARTED = 2 # could not instantiate an experiment to run the test on
+
+ def run(self):
+ powder_host = PowderProfile()
+ # expect the start on a specified profile succeeds and return
+ # an IPv4 address to proceed to the next step
+ status, ip_address = powder_host.run()
+
+ if not ip_address:
+ sys.exit(self.TEST_NOT_STARTED)
+ elif self._start_powder_experiment(ip_address):
+ sys.exit(self.TEST_FAILED)
+ else:
+ sys.exit(self.TEST_SUCCEEDED)
+
+ def _start_powder_experiment(self, ip_address):
+ logging.info('Executing ssh commands on host:{}'.format(ip_address))
+ node = pexp.Node(ip_address=ip_address)
+ ssh_node = node.ssh.open()
+ # the example commands shown below are to set up the AI/ML FW from scratch
+ ssh_node.command('sudo groupadd docker && sudo usermod -aG docker osc_int')
+ ssh_node.close(5)
+ ssh_node = node.ssh.open()
+ ssh_node.command('git clone https://gerrit.o-ran-sc.org/r/aiml-fw/aimlfw-dep')
+ ssh_node.command('cd aimlfw-dep && bin/install_traininghost.sh 2>&1 | tee /tmp/install.log', timeout=1800)
+ ssh_node.close(5)
+
+if __name__ == '__main__':
+ powdertest = RunTest()
+ powdertest.run()