From: pceicicd Date: Thu, 10 Nov 2022 23:59:17 +0000 (+0000) Subject: create a XTesting directory to first add the required files for a RIC platform deploy... X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=a1d10aced510b9794aee3006dc12a97f7514bcae;p=it%2Ftest.git create a XTesting directory to first add the required files for a RIC platform deployment and healthcheck work flow demonstration Change-Id: I8a67499bfda21f51b8a30dc44f1b0904d4ccf222 Signed-off-by: pceicicd --- diff --git a/XTesting/XTesting-demo/README.md b/XTesting/XTesting-demo/README.md new file mode 100644 index 0000000..7f84b35 --- /dev/null +++ b/XTesting/XTesting-demo/README.md @@ -0,0 +1,8 @@ +# XTesting-demo + +This repo is to demonstrate the XTesting work flow against the deployment of the OSC RIC platform followed up with a health check test case. + +To run the demo, clone the repo on the XTesting host and run the following command: + +sudo ./install_dependencies.sh # if the XTesting is not yet set up on the host +sudo ./demo.sh target-ip private-key-file-path [working-directory] diff --git a/XTesting/XTesting-demo/demo.sh b/XTesting/XTesting-demo/demo.sh new file mode 100755 index 0000000..8cac36a --- /dev/null +++ b/XTesting/XTesting-demo/demo.sh @@ -0,0 +1,90 @@ +#! /bin/bash +set -x + +echo "This is to demonstrate the XTesting work flow against the OSC RIC platform deployment and perform health check" + +if [ $# -lt 2 ] +then + echo "Usage: $0 target-ip private-key-file-path [working-directory]" + exit 1 +fi + +# pick up the input parameters from command line +IP=$1 +KEYFILE=$2 +ORIG=$PWD +if [ $# -ge 3 ] +then + WORKDIR=$3 +else + WORKDIR=$PWD +fi + +# make the working directory if it's not there yet +if [ ! -d $WORKDIR ] +then + mkdir -p $WORKDIR +fi + +# copy over the health check test case to the working directory +cp healthcheck.robot $WORKDIR + +cd $WORKDIR + +# replace it with the target IP address for health check +sed -i "s/TARGET-IP/${IP}/" healthcheck.robot + +###### step 1 deploy Kubernetes cluster and obtain the Kube config + +# remove the old one if it's already there +if [ -d kubeadm ] +then + rm -rf kubeadm +fi + +# copy over the content to build the kubeadm container +mkdir kubeadm +cp -r $ORIG/../kubeadm/* kubeadm/ + +cd kubeadm +# update the host IP in the environment file +TMPFILE=/tmp/tmp`date +%s` +cat sample_env | sed -e '/ANSIBLE_HOST_IP/d' > $TMPFILE +echo "ANSIBLE_HOST_IP=${IP}" > sample_env +cat $TMPFILE >> sample_env + +# copy the private key to the inventory/sample folder as id_rsa +cp $KEYFILE id_rsa +chmod 400 id_rsa + +# build the Docker image +docker build -t kubeadm . + +# run the docker container to deploy Kubernetes onto the SUT specified by the IP address +docker run -v ~/.kube:/kubeadm/config kubeadm + +###### step 2 complete the deployment based on the Kube config from step 1 + +cd $WORKDIR +# remove the old one if it's already there +if [ -d richelm ] +then + rm -rf richelm +fi + +#copy over the content to build the richelm container +mkdir richelm +cp -r $ORIG/../richelm/* richelm/ + +#build the container +cd richelm && ./build.sh static + +#run the container with the kubeconfig to complete the RIC platform deployment +sudo docker run -ti --rm -w /apps -v ~/.kube:/root/.kube -t richelmlegacy:1.19.16 + +# sometimes some RIC platform containers are not up right away so wait a bit +sleep 60 + +cd $WORKDIR +###### step 3 run the health check test case to complete the demo +ansible-playbook healthcheck.robot diff --git a/XTesting/XTesting-demo/healthcheck.robot b/XTesting/XTesting-demo/healthcheck.robot new file mode 100644 index 0000000..7014c0a --- /dev/null +++ b/XTesting/XTesting-demo/healthcheck.robot @@ -0,0 +1,12 @@ +--- +- name: Shell module example + hosts: 127.0.0.1 + tasks: + + - name: Check system information + shell: + "curl -v http://TARGET-IP:32080/appmgr/ric/v1/health/ready 2>&1" + register: os_info + + - debug: + msg: "{{os_info.stdout_lines}}" diff --git a/XTesting/XTesting-demo/install-dependencies.sh b/XTesting/XTesting-demo/install-dependencies.sh new file mode 100755 index 0000000..7238571 --- /dev/null +++ b/XTesting/XTesting-demo/install-dependencies.sh @@ -0,0 +1,35 @@ +#! /bin/bash -x + +# openssh +apt install -y openssh-server +systemctl status ssh +ufw allow ssh + +# docker +apt-get remove docker docker-engine docker.io containerd runc +apt-get update +apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release + +mkdir -p /etc/apt/keyrings + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + + +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin +systemctl enable docker +systemctl start docker + +# XTesting dependencies +apt update && apt install git -y +[ -z "$VIRTUAL_ENV" ] && apt install python3-pip -y && pip3 install ansible +ansible-galaxy install collivier.xtesting +ansible-galaxy collection install ansible.posix community.general community.grafana \ + community.kubernetes community.docker community.postgresql diff --git a/XTesting/kubeadm/Dockerfile b/XTesting/kubeadm/Dockerfile new file mode 100644 index 0000000..83cdbeb --- /dev/null +++ b/XTesting/kubeadm/Dockerfile @@ -0,0 +1,31 @@ +# Use imutable image tags rather than mutable tags (like ubuntu:20.04) +FROM ubuntu:focal-20220531 + +ARG ARCH=amd64 +ARG KUBE_VERSION=v1.19.16 +ARG TZ=Etc/UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt update -y \ + && apt install -y \ + libssl-dev python3-dev sshpass apt-transport-https jq moreutils \ + ca-certificates curl gnupg2 software-properties-common python3-pip unzip rsync git \ + && rm -rf /var/lib/apt/lists/* + + +ENV LANG=C.UTF-8 + +WORKDIR /kubeadm +COPY . . + +RUN /usr/bin/python3 -m pip install --no-cache-dir pip -U \ + && python3 -m pip install --no-cache-dir -r requirements.txt \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$ARCH/kubectl \ + && chmod a+x kubectl \ + && mv kubectl /usr/local/bin/kubectl + +RUN chmod a+x deploy.sh + +CMD /kubeadm/deploy.sh diff --git a/XTesting/kubeadm/cluster.yml b/XTesting/kubeadm/cluster.yml new file mode 100644 index 0000000..15db72c --- /dev/null +++ b/XTesting/kubeadm/cluster.yml @@ -0,0 +1,132 @@ +- hosts: all + become: true + tasks: + - name: Install packages that allow apt to be used over HTTPS + apt: + name: "{{ packages }}" + state: present + vars: + packages: + - apt-transport-https + - ca-certificates + - curl + - gnupg-agent + - software-properties-common + + - name: Add the apt signing key for Docker + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + + - name: Add the apt repository for the stable Docker version + apt_repository: + repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable + state: present + + - name: Install Docker and its dependencies + apt: + name: "{{ packages }}" + state: present + update_cache: yes + vars: + packages: + - docker-ce + - docker-ce-cli + - containerd.io + + - name: Ensure user group exists + group: + name: "{{ ansible_user }}" + + - name: Add user to the docker group + user: + name: "{{ ansible_user }}" + groups: docker,{{ ansible_user }} + append: yes + + - name: Remove swapfile from /etc/fstab + mount: + name: "{{ item }}" + fstype: swap + state: absent + with_items: + - swap + - none + + - name: Disable swap + command: swapoff -a + when: ansible_swaptotal_mb > 0 + + - name: Add the apt signing key for Kubernetes + apt_key: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + state: present + + - name: Add the apt repository for Kubernetes + apt_repository: + repo: deb https://apt.kubernetes.io/ kubernetes-xenial main + state: present + filename: kubernetes.list + + - name: Install Kubernetes binaries + apt: + name: "{{ packages }}" + update_cache: yes + vars: + packages: + - kubelet=1.19.16-00 + - kubeadm=1.19.16-00 + - kubectl=1.19.16-00 + + - name: Restart kubelet + service: + name: kubelet + daemon_reload: yes + state: restarted + + - name: Making systemd as Cgroup Driver + copy: + src: daemon.json + dest: /etc/docker/daemon.json + + - name: Restaring Docker... + service: + name: docker + state: restarted + + - name: update sysctl param + copy: + src: k8s.conf + dest: /etc/sysctl.d/k8s.conf + owner: root + group: root + + - name: System reload + shell: sysctl --system + + - name: Initialize the Kubernetes cluster + become: true + command: kubeadm init --pod-network-cidr=10.244.0.0/16 + + - name: Setup kubeconfig + command: "{{ item }}" + with_items: + - mkdir -p "$HOME/.kube" + - sudo cp /etc/kubernetes/admin.conf "$HOME/.kube/config" + - sudo chown "{{ ansible_user }}:{{ ansible_user }}" "$HOME/.kube/config" + + - name: Deploy Flannel + become: true + command: kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml + + - name: Remove taints from master node + become: true + command: kubectl taint nodes --all node-role.kubernetes.io/master- + + - name: Create Tiller service account + become: true + command: kubectl -n kube-system create serviceaccount tiller + + - name: Setup Tiller ClusterBinding + become: true + command: kubectl create clusterrolebinding tiller --clusterrole=cluster-admin --serviceaccount=kube-system:tiller diff --git a/XTesting/kubeadm/daemon.json b/XTesting/kubeadm/daemon.json new file mode 100644 index 0000000..5d18abc --- /dev/null +++ b/XTesting/kubeadm/daemon.json @@ -0,0 +1,8 @@ +{ + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, + "storage-driver": "overlay2" +} diff --git a/XTesting/kubeadm/deploy.sh b/XTesting/kubeadm/deploy.sh new file mode 100644 index 0000000..198f69e --- /dev/null +++ b/XTesting/kubeadm/deploy.sh @@ -0,0 +1,12 @@ +#!/bin/bash -x + +#cp -rfp inventory/sample inventory/oransc-cluster +. sample_env +declare -a IPS=($ANSIBLE_HOST_IP) +CONFIG_FILE=hosts.yaml python3 inventory.py ${IPS[@]} +export ANSIBLE_HOST_KEY_CHECKING=False +ansible-playbook -i hosts.yaml --become --private-key ${ANSIBLE_SSH_KEY} cluster.yml + +#sshpass -p $ANSIBLE_PASSWORD scp -o StrictHostKeyChecking=no -i ${ANSIBLE_SSH_KEY} -q root@$ANSIBLE_HOST_IP:/root/.kube/config ${PROJECT_ROOT}/config +scp -o StrictHostKeyChecking=no -i ${ANSIBLE_SSH_KEY} -q root@$ANSIBLE_HOST_IP:/root/.kube/config ${PROJECT_ROOT}/config +sed -i "s/127.0.0.1/${ANSIBLE_HOST_IP}/g" "${PROJECT_ROOT}"/config diff --git a/XTesting/kubeadm/inventory.py b/XTesting/kubeadm/inventory.py new file mode 100644 index 0000000..0a00599 --- /dev/null +++ b/XTesting/kubeadm/inventory.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Usage: inventory.py ip1 [ip2 ...] +# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 +# +# Advanced usage: +# Add another host after initial creation: inventory.py 10.10.1.5 +# Add range of hosts: inventory.py 10.10.1.3-10.10.1.5 +# Add hosts with different ip and access ip: +# inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.1.3 +# Add hosts with a specific hostname, ip, and optional access ip: +# inventory.py first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3 +# Delete a host: inventory.py -10.10.1.3 +# Delete a host by id: inventory.py -node1 +# +# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml +# YAML file should be in the following format: +# group1: +# host1: +# ip: X.X.X.X +# var: val +# group2: +# host2: +# ip: X.X.X.X + +from collections import OrderedDict +from ipaddress import ip_address +from ruamel.yaml import YAML + +import os +import re +import subprocess +import sys + +ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster', + 'calico_rr'] +PROTECTED_NAMES = ROLES +AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames', + 'load', 'add'] +_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False} +yaml = YAML() +yaml.Representer.add_representer(OrderedDict, yaml.Representer.represent_dict) + + +def get_var_as_bool(name, default): + value = os.environ.get(name, '') + return _boolean_states.get(value.lower(), default) + +# Configurable as shell vars start + + +CONFIG_FILE = os.environ.get("CONFIG_FILE", "./hosts.yaml") +# Remove the reference of KUBE_MASTERS after some deprecation cycles. +KUBE_CONTROL_HOSTS = int(os.environ.get("KUBE_CONTROL_HOSTS", + os.environ.get("KUBE_MASTERS", 2))) +# Reconfigures cluster distribution at scale +SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50)) +MASSIVE_SCALE_THRESHOLD = int(os.environ.get("MASSIVE_SCALE_THRESHOLD", 200)) + +DEBUG = get_var_as_bool("DEBUG", True) +HOST_PREFIX = os.environ.get("HOST_PREFIX", "node") +USE_REAL_HOSTNAME = get_var_as_bool("USE_REAL_HOSTNAME", False) + +# Configurable as shell vars end + + +class KubesprayInventory(object): + + def __init__(self, changed_hosts=None, config_file=None): + self.config_file = config_file + self.yaml_config = {} + loadPreviousConfig = False + printHostnames = False + # See whether there are any commands to process + if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS: + if changed_hosts[0] == "add": + loadPreviousConfig = True + changed_hosts = changed_hosts[1:] + elif changed_hosts[0] == "print_hostnames": + loadPreviousConfig = True + printHostnames = True + else: + self.parse_command(changed_hosts[0], changed_hosts[1:]) + sys.exit(0) + + # If the user wants to remove a node, we need to load the config anyway + if changed_hosts and changed_hosts[0][0] == "-": + loadPreviousConfig = True + + if self.config_file and loadPreviousConfig: # Load previous YAML file + try: + self.hosts_file = open(config_file, 'r') + self.yaml_config = yaml.load(self.hosts_file) + except OSError as e: + # I am assuming we are catching "cannot open file" exceptions + print(e) + sys.exit(1) + + if printHostnames: + self.print_hostnames() + sys.exit(0) + + self.ensure_required_groups(ROLES) + + if changed_hosts: + changed_hosts = self.range2ips(changed_hosts) + self.hosts = self.build_hostnames(changed_hosts, + loadPreviousConfig) + self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES) + self.set_all(self.hosts) + self.set_k8s_cluster() + etcd_hosts_count = 3 if len(self.hosts.keys()) >= 3 else 1 + self.set_etcd(list(self.hosts.keys())[:etcd_hosts_count]) + if len(self.hosts) >= SCALE_THRESHOLD: + self.set_kube_control_plane(list(self.hosts.keys())[ + etcd_hosts_count:(etcd_hosts_count + KUBE_CONTROL_HOSTS)]) + else: + self.set_kube_control_plane( + list(self.hosts.keys())[:KUBE_CONTROL_HOSTS]) + self.set_kube_node(self.hosts.keys()) + if len(self.hosts) >= SCALE_THRESHOLD: + self.set_calico_rr(list(self.hosts.keys())[:etcd_hosts_count]) + else: # Show help if no options + self.show_help() + sys.exit(0) + + self.write_config(self.config_file) + + def write_config(self, config_file): + if config_file: + with open(self.config_file, 'w') as f: + yaml.dump(self.yaml_config, f) + + else: + print("WARNING: Unable to save config. Make sure you set " + "CONFIG_FILE env var.") + + def debug(self, msg): + if DEBUG: + print("DEBUG: {0}".format(msg)) + + def get_ip_from_opts(self, optstring): + if 'ip' in optstring: + return optstring['ip'] + else: + raise ValueError("IP parameter not found in options") + + def ensure_required_groups(self, groups): + for group in groups: + if group == 'all': + self.debug("Adding group {0}".format(group)) + if group not in self.yaml_config: + all_dict = OrderedDict([('hosts', OrderedDict({})), + ('children', OrderedDict({}))]) + self.yaml_config = {'all': all_dict} + else: + self.debug("Adding group {0}".format(group)) + if group not in self.yaml_config['all']['children']: + self.yaml_config['all']['children'][group] = {'hosts': {}} + + def get_host_id(self, host): + '''Returns integer host ID (without padding) from a given hostname.''' + try: + short_hostname = host.split('.')[0] + return int(re.findall("\\d+$", short_hostname)[-1]) + except IndexError: + raise ValueError("Host name must end in an integer") + + # Keeps already specified hosts, + # and adds or removes the hosts provided as an argument + def build_hostnames(self, changed_hosts, loadPreviousConfig=False): + existing_hosts = OrderedDict() + highest_host_id = 0 + # Load already existing hosts from the YAML + if loadPreviousConfig: + try: + for host in self.yaml_config['all']['hosts']: + # Read configuration of an existing host + hostConfig = self.yaml_config['all']['hosts'][host] + existing_hosts[host] = hostConfig + # If the existing host seems + # to have been created automatically, detect its ID + if host.startswith(HOST_PREFIX): + host_id = self.get_host_id(host) + if host_id > highest_host_id: + highest_host_id = host_id + except Exception as e: + # I am assuming we are catching automatically + # created hosts without IDs + print(e) + sys.exit(1) + + # FIXME(mattymo): Fix condition where delete then add reuses highest id + next_host_id = highest_host_id + 1 + next_host = "" + + username = os.environ.get("ANSIBLE_USER", 'osc_int') + password = os.environ.get("ANSIBLE_PASSWORD", 'osc_int') + + + all_hosts = existing_hosts.copy() + for host in changed_hosts: + # Delete the host from config the hostname/IP has a "-" prefix + if host[0] == "-": + realhost = host[1:] + if self.exists_hostname(all_hosts, realhost): + self.debug("Marked {0} for deletion.".format(realhost)) + all_hosts.pop(realhost) + elif self.exists_ip(all_hosts, realhost): + self.debug("Marked {0} for deletion.".format(realhost)) + self.delete_host_by_ip(all_hosts, realhost) + # Host/Argument starts with a digit, + # then we assume its an IP address + elif host[0].isdigit(): + if ',' in host: + ip, access_ip = host.split(',') + else: + ip = host + access_ip = host + if self.exists_hostname(all_hosts, host): + self.debug("Skipping existing host {0}.".format(host)) + continue + elif self.exists_ip(all_hosts, ip): + self.debug("Skipping existing host {0}.".format(ip)) + continue + + if USE_REAL_HOSTNAME: + cmd = ("ssh -oStrictHostKeyChecking=no " + + access_ip + " 'hostname -s'") + next_host = subprocess.check_output(cmd, shell=True) + next_host = next_host.strip().decode('ascii') + else: + # Generates a hostname because we have only an IP address + next_host = "{0}{1}".format(HOST_PREFIX, next_host_id) + next_host_id += 1 + # Uses automatically generated node name + # in case we dont provide it. + if os.getenv('ANSIBLE_SSH_KEY'): + all_hosts[next_host] = {'ansible_host': access_ip, + 'ansible_user': username, + 'ansible_ssh_private_key_file': os.getenv('ANSIBLE_SSH_KEY'), + 'ip': ip, + 'access_ip': access_ip} + else: + all_hosts[next_host] = {'ansible_host': access_ip, + 'ansible_user': username, + 'ansible_password': password, + 'ip': ip, + 'access_ip': access_ip} + # Host/Argument starts with a letter, then we assume its a hostname + elif host[0].isalpha(): + if ',' in host: + try: + hostname, ip, access_ip = host.split(',') + except Exception: + hostname, ip = host.split(',') + access_ip = ip + if self.exists_hostname(all_hosts, host): + self.debug("Skipping existing host {0}.".format(host)) + continue + elif self.exists_ip(all_hosts, ip): + self.debug("Skipping existing host {0}.".format(ip)) + continue + all_hosts[hostname] = {'ansible_host': access_ip, + 'ip': ip, + 'access_ip': access_ip} + return all_hosts + + # Expand IP ranges into individual addresses + def range2ips(self, hosts): + reworked_hosts = [] + + def ips(start_address, end_address): + try: + # Python 3.x + start = int(ip_address(start_address)) + end = int(ip_address(end_address)) + except Exception: + # Python 2.7 + start = int(ip_address(str(start_address))) + end = int(ip_address(str(end_address))) + return [ip_address(ip).exploded for ip in range(start, end + 1)] + + for host in hosts: + if '-' in host and not (host.startswith('-') or host[0].isalpha()): + start, end = host.strip().split('-') + try: + reworked_hosts.extend(ips(start, end)) + except ValueError: + raise Exception("Range of ip_addresses isn't valid") + else: + reworked_hosts.append(host) + return reworked_hosts + + def exists_hostname(self, existing_hosts, hostname): + return hostname in existing_hosts.keys() + + def exists_ip(self, existing_hosts, ip): + for host_opts in existing_hosts.values(): + if ip == self.get_ip_from_opts(host_opts): + return True + return False + + def delete_host_by_ip(self, existing_hosts, ip): + for hostname, host_opts in existing_hosts.items(): + if ip == self.get_ip_from_opts(host_opts): + del existing_hosts[hostname] + return + raise ValueError("Unable to find host by IP: {0}".format(ip)) + + def purge_invalid_hosts(self, hostnames, protected_names=[]): + for role in self.yaml_config['all']['children']: + if role != 'k8s_cluster' and self.yaml_config['all']['children'][role]['hosts']: # noqa + all_hosts = self.yaml_config['all']['children'][role]['hosts'].copy() # noqa + for host in all_hosts.keys(): + if host not in hostnames and host not in protected_names: + self.debug( + "Host {0} removed from role {1}".format(host, role)) # noqa + del self.yaml_config['all']['children'][role]['hosts'][host] # noqa + # purge from all + if self.yaml_config['all']['hosts']: + all_hosts = self.yaml_config['all']['hosts'].copy() + for host in all_hosts.keys(): + if host not in hostnames and host not in protected_names: + self.debug("Host {0} removed from role all".format(host)) + del self.yaml_config['all']['hosts'][host] + + def add_host_to_group(self, group, host, opts=""): + self.debug("adding host {0} to group {1}".format(host, group)) + if group == 'all': + if self.yaml_config['all']['hosts'] is None: + self.yaml_config['all']['hosts'] = {host: None} + self.yaml_config['all']['hosts'][host] = opts + elif group != 'k8s_cluster:children': + if self.yaml_config['all']['children'][group]['hosts'] is None: + self.yaml_config['all']['children'][group]['hosts'] = { + host: None} + else: + self.yaml_config['all']['children'][group]['hosts'][host] = None # noqa + + def set_kube_control_plane(self, hosts): + for host in hosts: + self.add_host_to_group('kube_control_plane', host) + + def set_all(self, hosts): + for host, opts in hosts.items(): + self.add_host_to_group('all', host, opts) + + def set_k8s_cluster(self): + k8s_cluster = {'children': {'kube_control_plane': None, + 'kube_node': None}} + self.yaml_config['all']['children']['k8s_cluster'] = k8s_cluster + + def set_calico_rr(self, hosts): + for host in hosts: + if host in self.yaml_config['all']['children']['kube_control_plane']: # noqa + self.debug("Not adding {0} to calico_rr group because it " + "conflicts with kube_control_plane " + "group".format(host)) + continue + if host in self.yaml_config['all']['children']['kube_node']: + self.debug("Not adding {0} to calico_rr group because it " + "conflicts with kube_node group".format(host)) + continue + self.add_host_to_group('calico_rr', host) + + def set_kube_node(self, hosts): + for host in hosts: + if len(self.yaml_config['all']['hosts']) >= SCALE_THRESHOLD: + if host in self.yaml_config['all']['children']['etcd']['hosts']: # noqa + self.debug("Not adding {0} to kube_node group because of " + "scale deployment and host is in etcd " + "group.".format(host)) + continue + if len(self.yaml_config['all']['hosts']) >= MASSIVE_SCALE_THRESHOLD: # noqa + if host in self.yaml_config['all']['children']['kube_control_plane']['hosts']: # noqa + self.debug("Not adding {0} to kube_node group because of " + "scale deployment and host is in " + "kube_control_plane group.".format(host)) + continue + self.add_host_to_group('kube_node', host) + + def set_etcd(self, hosts): + for host in hosts: + self.add_host_to_group('etcd', host) + + def load_file(self, files=None): + '''Directly loads JSON to inventory.''' + + if not files: + raise Exception("No input file specified.") + + import json + + for filename in list(files): + # Try JSON + try: + with open(filename, 'r') as f: + data = json.load(f) + except ValueError: + raise Exception("Cannot read %s as JSON, or CSV", filename) + + self.ensure_required_groups(ROLES) + self.set_k8s_cluster() + for group, hosts in data.items(): + self.ensure_required_groups([group]) + for host, opts in hosts.items(): + optstring = {'ansible_host': opts['ip'], + 'ip': opts['ip'], + 'access_ip': opts['ip']} + self.add_host_to_group('all', host, optstring) + self.add_host_to_group(group, host) + self.write_config(self.config_file) + + def parse_command(self, command, args=None): + if command == 'help': + self.show_help() + elif command == 'print_cfg': + self.print_config() + elif command == 'print_ips': + self.print_ips() + elif command == 'print_hostnames': + self.print_hostnames() + elif command == 'load': + self.load_file(args) + else: + raise Exception("Invalid command specified.") + + def show_help(self): + help_text = '''Usage: inventory.py ip1 [ip2 ...] +Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 + +Available commands: +help - Display this message +print_cfg - Write inventory file to stdout +print_ips - Write a space-delimited list of IPs from "all" group +print_hostnames - Write a space-delimited list of Hostnames from "all" group +add - Adds specified hosts into an already existing inventory + +Advanced usage: +Create new or overwrite old inventory file: inventory.py 10.10.1.5 +Add another host after initial creation: inventory.py add 10.10.1.6 +Add range of hosts: inventory.py 10.10.1.3-10.10.1.5 +Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3 +Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3 +Delete a host: inventory.py -10.10.1.3 +Delete a host by id: inventory.py -node1 + +Configurable env vars: +DEBUG Enable debug printing. Default: True +CONFIG_FILE File to write config to Default: ./inventory/sample/hosts.yaml +HOST_PREFIX Host prefix for generated hosts. Default: node +KUBE_CONTROL_HOSTS Set the number of kube-control-planes. Default: 2 +SCALE_THRESHOLD Separate ETCD role if # of nodes >= 50 +MASSIVE_SCALE_THRESHOLD Separate K8s control-plane and ETCD if # of nodes >= 200 +''' # noqa + print(help_text) + + def print_config(self): + yaml.dump(self.yaml_config, sys.stdout) + + def print_hostnames(self): + print(' '.join(self.yaml_config['all']['hosts'].keys())) + + def print_ips(self): + ips = [] + for host, opts in self.yaml_config['all']['hosts'].items(): + ips.append(self.get_ip_from_opts(opts)) + print(' '.join(ips)) + + +def main(argv=None): + if not argv: + argv = sys.argv[1:] + KubesprayInventory(argv, CONFIG_FILE) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/XTesting/kubeadm/k8s.conf b/XTesting/kubeadm/k8s.conf new file mode 100644 index 0000000..d46bab8 --- /dev/null +++ b/XTesting/kubeadm/k8s.conf @@ -0,0 +1,3 @@ +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +net.ipv4.ip_forward = 1 diff --git a/XTesting/kubeadm/requirements.txt b/XTesting/kubeadm/requirements.txt new file mode 100644 index 0000000..f8b1e50 --- /dev/null +++ b/XTesting/kubeadm/requirements.txt @@ -0,0 +1,10 @@ +ansible==5.7.1 +ansible-core==2.12.5 +cryptography==3.4.8 +jinja2==2.11.3 +netaddr==0.7.19 +pbr==5.4.4 +jmespath==0.9.5 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.6 +MarkupSafe==1.1.1 diff --git a/XTesting/kubeadm/sample_env b/XTesting/kubeadm/sample_env new file mode 100644 index 0000000..00c4891 --- /dev/null +++ b/XTesting/kubeadm/sample_env @@ -0,0 +1,5 @@ +ANSIBLE_HOST_IP=155.98.36.91 +ANSIBLE_USERNAME=osc_int +ANSIBLE_PASSWORD=osc_int +PROJECT_ROOT=/kubeadm/config +ANSIBLE_SSH_KEY=id_rsa diff --git a/XTesting/richelm/Dockerfile b/XTesting/richelm/Dockerfile new file mode 100644 index 0000000..b07c377 --- /dev/null +++ b/XTesting/richelm/Dockerfile @@ -0,0 +1,74 @@ +FROM alpine + +# Ignore to update versions here +# docker build --no-cache --build-arg KUBECTL_VERSION=${tag} --build-arg HELM_VERSION=${helm} --build-arg KUSTOMIZE_VERSION=${kustomize_version} -t ${image}:${tag} . +ARG HELM_VERSION=2.17.0 +ARG KUBECTL_VERSION=1.19.16 +ARG KUSTOMIZE_VERSION=v3.8.1 +ARG KUBESEAL_VERSION=0.18.1 + +# Install helm (latest release) +# ENV BASE_URL="https://storage.googleapis.com/kubernetes-helm" +ENV BASE_URL="https://get.helm.sh" +ENV TAR_FILE="helm-v${HELM_VERSION}-linux-amd64.tar.gz" +RUN apk add --update --no-cache curl ca-certificates bash git && \ + curl -sL ${BASE_URL}/${TAR_FILE} | tar -xvz && \ + mv linux-amd64/helm /usr/bin/helm && \ + chmod +x /usr/bin/helm && \ + rm -rf linux-amd64 + +RUN helm init --client-only + +# add helm-diff +RUN helm plugin install https://github.com/databus23/helm-diff && rm -rf /tmp/helm-* + +# add helm-unittest +RUN helm plugin install https://github.com/quintush/helm-unittest && rm -rf /tmp/helm-* + +# add helm-push +RUN helm plugin install https://github.com/chartmuseum/helm-push && rm -rf /tmp/helm-* + +# Install kubectl (same version of aws esk) +RUN curl -sLO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ + mv kubectl /usr/bin/kubectl && \ + chmod +x /usr/bin/kubectl + +# Install kustomize (latest release) +RUN curl -sLO https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + tar xvzf kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + mv kustomize /usr/bin/kustomize && \ + chmod +x /usr/bin/kustomize + +# Install eksctl (latest version) +RUN curl -sL "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && \ + mv /tmp/eksctl /usr/bin && \ + chmod +x /usr/bin/eksctl + +# Install awscli +RUN apk add --update --no-cache python3 && \ + python3 -m ensurepip && \ + pip3 install --upgrade pip && \ + pip3 install awscli && \ + pip3 cache purge + +# https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html +# Install aws-iam-authenticator +RUN authenticator=$(aws --no-sign-request s3 ls s3://amazon-eks --recursive |grep aws-iam-authenticator$|grep amd64 |awk '{print $NF}' |sort -V|tail -1) && \ + aws --no-sign-request s3 cp s3://amazon-eks/${authenticator} /usr/bin/aws-iam-authenticator && \ + chmod +x /usr/bin/aws-iam-authenticator + +# Install jq +RUN apk add --update --no-cache jq yq + +# Install for envsubst +RUN apk add --update --no-cache gettext + +# Install kubeseal +RUN curl -L https://github.com/bitnami-labs/sealed-secrets/releases/download/v${KUBESEAL_VERSION}/kubeseal-${KUBESEAL_VERSION}-linux-amd64.tar.gz -o - | tar xz -C /usr/bin/ && \ + chmod +x /usr/bin/kubeseal + +WORKDIR /apps +COPY . . + +RUN chmod a+x deploy.sh +CMD ./deploy.sh diff --git a/XTesting/richelm/README.md b/XTesting/richelm/README.md new file mode 100644 index 0000000..1c4e86c --- /dev/null +++ b/XTesting/richelm/README.md @@ -0,0 +1,38 @@ +# Kubernetes tools for RIC CI/CD + +kubernetes images with necessary tools that can be used as normal kubectl tool along with AWS EKS. + +### Installed tools + +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) (eks versions: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html) +- [kustomize](https://github.com/kubernetes-sigs/kustomize) (latest release: https://github.com/kubernetes-sigs/kustomize/releases/latest) +- [helm](https://github.com/helm/helm) (latest release: https://github.com/helm/helm/releases/latest) +- [helm-diff](https://github.com/databus23/helm-diff) (latest commit) +- [helm-unittest](https://github.com/quintush/helm-unittest) (latest commit) +- [helm-push](https://github.com/chartmuseum/helm-push) (latest commit) +- [aws-iam-authenticator](https://github.com/kubernetes-sigs/aws-iam-authenticator) (latest version when run the build) +- [eksctl](https://github.com/weaveworks/eksctl) (latest version when run the build) +- [awscli v1](https://github.com/aws/aws-cli) (latest version when run the build) +- [kubeseal](https://github.com/bitnami-labs/sealed-secrets) (latest version when run the build) +- General tools, such as bash, curl + +# Why we need it + +Mostly it is used during CI/CD (continuous integration and continuous delivery) or as part of an automated build/deployment + +# kubectl versions + +You should check in [kubernetes versions](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html), it lists the kubectl version and used as image tags. + +# Involve with developing and testing + +If you want to build these images by yourself, please follow below commands. + +``` +./build.sh static +``` + +# Usage + + # mount local folder with kube config in container. + docker run -ti --rm -w /apps -v ~/.kube:/root/.kube -t richelmlegacy:1.19.16 diff --git a/XTesting/richelm/build.sh b/XTesting/richelm/build.sh new file mode 100755 index 0000000..3e19e90 --- /dev/null +++ b/XTesting/richelm/build.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +# Prerequisite +# Make sure you set secret enviroment variables in CI +# DOCKER_USERNAME +# DOCKER_PASSWORD + +# set -ex + +set -e + +build() { + + # helm latest + helm=$(curl -s https://github.com/helm/helm/releases) + helm=$(echo $helm\" |grep -oP '(?<=tag\/v)[0-9][^"]*'|grep -v \-|sort -Vr|head -1) + echo "helm version is $helm" + + # jq 1.6 + DEBIAN_FRONTEND=noninteractive + #sudo apt-get update && sudo apt-get -q -y install jq + curl -sL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o jq + sudo mv jq /usr/bin/jq + sudo chmod +x /usr/bin/jq + + # kustomize latest + kustomize_release=$(curl -s https://api.github.com/repos/kubernetes-sigs/kustomize/releases | /usr/bin/jq -r '.[].tag_name | select(contains("kustomize"))' \ + | sort -rV | head -n 1) + kustomize_version=$(basename ${kustomize_release}) + echo "kustomize version is $kustomize_version" + + # kubeseal latest + kubeseal_version=$(curl -s https://api.github.com/repos/bitnami-labs/sealed-secrets/releases | /usr/bin/jq -r '.[].tag_name | select(startswith("v"))' \ + | sort -rV | head -n 1 |sed 's/v//') + echo "kubeseal version is $kubeseal_version" + + docker build --no-cache \ + --build-arg KUBECTL_VERSION=${tag} \ + --build-arg HELM_VERSION=${helm} \ + --build-arg KUSTOMIZE_VERSION=${kustomize_version} \ + --build-arg KUBESEAL_VERSION=${kubeseal_version} \ + -t ${image}:${tag} . + + # run test + echo "Detected Helm3+" + version=$(docker run --rm ${image}:${tag} helm version) + # version.BuildInfo{Version:"v3.6.3", GitCommit:"d506314abfb5d21419df8c7e7e68012379db2354", GitTreeState:"clean", GoVersion:"go1.16.5"} + + version=$(echo ${version}| awk -F \" '{print $2}') + if [ "${version}" == "v${helm}" ]; then + echo "matched" + else + echo "unmatched" + exit + fi + + if [[ "$CIRCLE_BRANCH" == "master" ]]; then + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + docker push ${image}:${tag} + fi +} + +build_static() { + docker build --no-cache \ + -t ${image}:${tag} . + +} + +image="richelmlegacy" +bimage="alpine/k8s" +if [ $# -eq 0 ] + then + curl -s https://raw.githubusercontent.com/awsdocs/amazon-eks-user-guide/master/doc_source/kubernetes-versions.md |egrep -A 10 "The following Kubernetes versions"|awk -F \` '/^\+/ {print $2}'|sort -Vr | while read tag + do + echo ${tag} + status=$(curl -sL https://hub.docker.com/v2/repositories/${bimage}/tags/${tag}) + echo $status + if [[ ( "${status}" =~ "not found" ) || ( ${REBUILD} == "true" ) ]]; then + build + break + fi + done + else + tag=$(awk -F '=' '/ARG\ KUBECTL_VERSION/{print $NF}' Dockerfile) + echo ${tag} + build_static +fi diff --git a/XTesting/richelm/deploy.sh b/XTesting/richelm/deploy.sh new file mode 100644 index 0000000..c46bb34 --- /dev/null +++ b/XTesting/richelm/deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -x + +# initiate the tiller +helm init --service-account tiller +git clone "https://gerrit.o-ran-sc.org/r/ric-plt/ric-dep" + +export VERIFY_CHECKSUM=false +cd ric-dep/bin && ./verify-ric-charts && cat ../RECIPE_EXAMPLE/example_recipe_oran_f_release.yaml | sed -e 's/10\.0\.0\.1//g' > ../RECIPE_EXAMPLE/example_recipe_oran_f_release.yaml.overwrite && cat install | sed -e 's/bash/bash -x/' | sed -e 's/helm install/helm install --debug/' > install2 && chmod +x install2 && ./install2 -f ../RECIPE_EXAMPLE/example_recipe_oran_f_release.yaml.overwrite