xapp_orchestrater/dev/xapp_onboarder/xapp_onboarder/helm_controller/xApp_builder.py

   1 ################################################################################
   2 #   Copyright (c) 2020 AT&T Intellectual Property.                             #
   3 #                                                                              #
   4 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   5 #   you may not use this file except in compliance with the License.           #
   6 #   You may obtain a copy of the License at                                    #
   7 #                                                                              #
   8 #       http://www.apache.org/licenses/LICENSE-2.0                             #
   9 #                                                                              #
  10 #   Unless required by applicable law or agreed to in writing, software        #
  11 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  12 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  13 #   See the License for the specific language governing permissions and        #
  14 #   limitations under the License.                                             #
  15 ################################################################################
  16
  17 import logging
  18 import yaml
  19 import json
  20 import os
  21 import io
  22 import subprocess
  23 import shutil
  24 import re
  25 import copy
  26 import platform
  27 import tarfile
  28 import stat
  29 import sys
  30 from xapp_onboarder.server import settings
  31 from xapp_onboarder.repo_manager.repo_manager import repo_manager, RepoManagerError
  32 from pkg_resources import resource_filename
  33 from subprocess import PIPE, check_output, STDOUT
  34 from xapp_onboarder.repo_manager.repo_manager import requests_retry_session
  35 log = logging.getLogger(__name__)
  36
  37
  38 def indent(text, amount, ch=' '):
  39     padding = amount * ch
  40     return ''.join(padding + line for line in text.splitlines(True))
  41
  42
  43 class xAppError(Exception):
  44     def __init__(self, message, status_code):
  45         # Call the base class constructor with the parameters it needs
  46         super().__init__(message)
  47         self.status_code = status_code
  48
  49
  50 class xApp():
  51     def __init__(self, config_file, schema_file):
  52         self.config_file = config_file
  53         self.schema_file = schema_file
  54
  55         isnamepresent = 0
  56         if 'name' not in self.config_file:
  57             isnamepresent = 1
  58             if 'xapp_name' not in self.config_file:
  59                 raise xAppError(
  60                     "xApp chart name not found. (Caused by: config-file.json does not contain xapp_name attribute.)", 500)
  61
  62         if 'version' not in self.config_file:
  63             raise xAppError(
  64                 "xApp chart version not found. (Caused by: config-file.json does not contain version attribute.)", 500)
  65
  66         if isnamepresent == 1:
  67             self.chart_name = self.config_file['xapp_name']
  68         else:
  69             self.chart_name = self.config_file['name']
  70         self.chart_version = self.config_file['version']
  71         self.configmap_config_json_file = copy.deepcopy(self.config_file)
  72         self.chart_workspace_path = settings.CHART_WORKSPACE_PATH + '/' + self.chart_name + '-' + self.chart_version
  73         if os.path.exists(self.chart_workspace_path):
  74             shutil.rmtree(self.chart_workspace_path)
  75         os.makedirs(self.chart_workspace_path)
  76         shutil.copytree(resource_filename( 'xapp_onboarder', 'resources/xapp-std'), self.chart_workspace_path + '/' + self.chart_name)
  77         self.setup_helm()
  78
  79     def setup_helm(self):
  80         self.helm_client_path = 'helm'
  81         try:
  82             process = subprocess.run([self.helm_client_path], stdout=PIPE, stderr=PIPE, check=True)
  83
  84         except Exception as err:
  85             print(err)
  86             self.download_helm()
  87             self.helm_client_path = settings.CHART_WORKSPACE_PATH + '/helm'
  88
  89     def download_helm(self):
  90         if not os.path.isfile(settings.CHART_WORKSPACE_PATH + '/helm'):
  91             log.info("Helm client missing. Trying to download it.")
  92             helm_file_name = "helm-v{}-{}-amd64.tar.gz".format(settings.HELM_VERSION, platform.system().lower())
  93             helm_download_link = "https://get.helm.sh/" + helm_file_name
  94
  95
  96             try:
  97                 response = requests_retry_session().get(helm_download_link, timeout=settings.HTTP_TIME_OUT)
  98             except Exception as err:
  99                 error_message = "Download helm client failed. (Caused by: " + str(err)+")"
 100                 log.error(error_message)
 101                 raise xAppError(error_message, 500)
 102             else:
 103                 if response.status_code != 200:
 104                     error_message = "Download helm chart failed. Helm repo return status code: "+ str(response.status_code)  +" "+ response.content.decode("utf-8")
 105                     log.error(error_message)
 106                     raise xAppError(error_message, 500)
 107
 108                 file_stream = io.BytesIO(response.content)
 109
 110                 with tarfile.open(fileobj=file_stream) as tar:
 111                     helm_client = tar.extractfile(platform.system().lower() + "-amd64/helm")
 112                     with open(settings.CHART_WORKSPACE_PATH+'/helm', 'wb') as file:
 113                         file.write(helm_client.read())
 114                 st = os.stat(settings.CHART_WORKSPACE_PATH+'/helm')
 115                 os.chmod(settings.CHART_WORKSPACE_PATH+'/helm', st.st_mode | stat.S_IEXEC)
 116
 117
 118
 119
 120     def recursive_convert_config_file(self, node_list=list()):
 121         current_node = self.configmap_config_json_file
 122         helm_value_path = '.Values'
 123         for node in node_list:
 124             current_node = current_node.get(node)
 125             helm_value_path = helm_value_path + ' ' + "\"" + node + "\""
 126
 127         if type(current_node) is not dict:
 128             raise TypeError("Recursive write was called on a leaf node.")
 129
 130         for item in current_node.keys():
 131             if type(current_node.get(item)) is not dict:
 132                 current_node[item] = '{{ index '+ helm_value_path +' "'+ item + '" | toJson }}'
 133             else:
 134                 new_node_list = node_list.copy()
 135                 new_node_list.append(item)
 136                 self.recursive_convert_config_file(new_node_list)
 137
 138
 139     def append_config_to_config_map(self):
 140         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/appconfig.yaml', 'a') as outputfile:
 141             self.recursive_convert_config_file()
 142             config_file_json_text = json.dumps(self.configmap_config_json_file, indent=4)
 143             indented_config_text = indent(config_file_json_text, 4)
 144             indented_config_text = re.sub(r"\"{{", '{{', indented_config_text)
 145             indented_config_text = re.sub(r"}}\"", '}}', indented_config_text)
 146             indented_config_text = re.sub(r"\\", '', indented_config_text)
 147             outputfile.write("  config-file.json: |\n")
 148             outputfile.write(indented_config_text)
 149             outputfile.write("\n  schema.json: |\n")
 150             schema_json = json.dumps(self.schema_file, indent=4)
 151             indented_schema_text = indent(schema_json, 4)
 152             outputfile.write(indented_schema_text)
 153
 154
 155 # This is a work around for the bronze release to be backward compatible to the previous xapp standard helm template
 156     def write_config_and_schema(self):
 157         os.makedirs(self.chart_workspace_path + '/' + self.chart_name + '/descriptors')
 158         os.makedirs(self.chart_workspace_path + '/' + self.chart_name + '/config')
 159         with open(self.chart_workspace_path + '/' + self.chart_name + '/descriptors/schema.json', 'w') as outfile:
 160             json.dump(self.schema_file, outfile)
 161         with open(self.chart_workspace_path + '/' + self.chart_name + '/config/config-file.json', 'w') as outfile:
 162             json.dump(self.config_file, outfile)
 163
 164
 165
 166     def add_probes_to_deployment(self):
 167         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/deployment.yaml', 'a') as outputfile:
 168
 169             for probes in ['readinessProbe', 'livenessProbe']:
 170                 if self.configmap_config_json_file.get(probes):
 171                     probe_definition = self.configmap_config_json_file.get(probes)
 172                     probe_definition_yaml = yaml.dump(probe_definition, width=1000)
 173
 174                     print(probe_definition_yaml)
 175
 176                     indented_probe_definition_yaml = indent(probe_definition_yaml, 12)
 177                     indented_probe_definition_yaml = re.sub(r" \| toJson", '', indented_probe_definition_yaml)
 178                     indented_probe_definition_yaml = re.sub(r"'", '', indented_probe_definition_yaml)
 179                     outputfile.write("          "+probes+":\n")
 180                     outputfile.write(indented_probe_definition_yaml)
 181
 182
 183     def append_config_to_values_yaml(self):
 184         with open(self.chart_workspace_path + '/' + self.chart_name + '/values.yaml', 'a') as outputfile:
 185             yaml.dump(self.config_file, outputfile, default_flow_style=False)
 186
 187
 188     def change_chart_name_version(self):
 189         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'r') as inputfile:
 190             self.chart_yaml = yaml.load(inputfile, Loader=yaml.FullLoader)
 191             self.chart_yaml['version'] = self.chart_version
 192             self.chart_yaml['name'] = self.chart_name
 193
 194         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'w') as outputfile:
 195             yaml.dump(self.chart_yaml, outputfile, default_flow_style=False)
 196
 197
 198     def helm_lint(self):
 199         try:
 200             process = subprocess.run([self.helm_client_path, "lint", self.chart_workspace_path + "/" + self.chart_name], stdout=PIPE, stderr=PIPE, check=True)
 201
 202         except OSError as err:
 203             raise xAppError(
 204                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " + str(
 205                     err) + ")", 500)
 206         except subprocess.CalledProcessError as err:
 207             raise xAppError(
 208                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " +
 209                 err.stderr.decode("utf-8") +  "\n" + err.stdout.decode("utf-8") + ")", 400)
 210
 211     def package_chart(self):
 212         self.write_config_and_schema()
 213         self.append_config_to_config_map()
 214         self.append_config_to_values_yaml()
 215         self.add_probes_to_deployment()
 216         self.change_chart_name_version()
 217         self.helm_lint()
 218         try:
 219             process = subprocess.run([self.helm_client_path, "package", self.chart_workspace_path + "/" + self.chart_name, "-d"
 220                                ,self.chart_workspace_path], stdout=PIPE, stderr=PIPE, check=True)
 221
 222         except OSError as err:
 223                 raise xAppError("xApp "+ self.chart_name+'-'+self.chart_version +" packaging failed. (Caused by: "+str(err) +")", 500)
 224         except subprocess.CalledProcessError as err:
 225             raise xAppError(
 226                 "xApp " + self.chart_name + '-' + self.chart_version + " packaging failed. (Caused by: " +
 227                     err.stderr.decode("utf-8") + ")", 500)
 228
 229
 230
 231     def distribute_chart(self):
 232         try:
 233             repo_manager.upload_chart(self)
 234         except RepoManagerError as err:
 235             raise xAppError( "xApp " + self.chart_name + '-' + self.chart_version + " distribution failed. (Caused by: " + str(err) + ")" , err.status_code)
 236
 237     def install_chart_package(xapp_chart_name, version, namespace, overridefile):
 238         dirTomove = "/tmp/helm_template"
 239         try:
 240           tar = tarfile.open(xapp_chart_name + "-" + version + ".tgz")
 241           tar.extractall()
 242           tar.close()
 243           if overridefile != "":
 244             process = subprocess.run(["helm", "install", xapp_chart_name, "./" + xapp_chart_name, "-f", overridefile, "--namespace=" + namespace], stdout=PIPE, stderr=PIPE, check=True)
 245           else:
 246             process = subprocess.run(["helm", "install", xapp_chart_name, "./" + xapp_chart_name, "--namespace=" + namespace], stdout=PIPE, stderr=PIPE, check=True)
 247           status = 1
 248         except subprocess.CalledProcessError as err:
 249             print(err.stderr.decode())
 250             status=0
 251         except Exception as err:
 252             print(err)
 253             status = 0
 254         if (os.getcwd() != dirTomove):
 255             subprocess.run(["mv", xapp_chart_name, dirTomove])
 256             PATH=xapp_chart_name + "-" + version + ".tgz"
 257             if os.path.isfile(PATH):
 258                 subprocess.run(["mv", xapp_chart_name + "-" + version + ".tgz", dirTomove ])
 259         return status
 260
 261     def uninstall_chart_package(xapp_chart_name, namespace, version):
 262         dirTomove = "/tmp/helm_template/"
 263         try:
 264           subprocess.run(["rm", "-rf", dirTomove + xapp_chart_name])
 265           if version != "" :
 266             subprocess.run(["rm", "-rf", dirTomove+xapp_chart_name + "-" + version + ".tgz"])
 267           process = subprocess.run(["helm", "delete", xapp_chart_name, "--namespace=" + namespace], stdout=PIPE, stderr=PIPE, check=True)
 268           status = 1
 269
 270         except Exception as err:
 271                 print(err.stderr.decode())
 272                 status = 0
 273
 274         return status
 275     def health_check_xapp(xapp_chart_name, namespace):
 276
 277         try:
 278           getpodname=subprocess.check_output("kubectl get po -n " + namespace + " |  grep -w " +  xapp_chart_name + " | awk '{print $1}'", shell=True).decode().strip("\n")
 279           if getpodname=="":
 280               print("No " + xapp_chart_name + " xapp found under " + namespace + " namespace.")
 281               sys.exit()
 282           process = subprocess.check_output("kubectl describe po " + getpodname +  " --namespace=" + namespace + "| grep -B 0 -A 5 'Conditions:'", shell=True).decode()
 283
 284           final= re.search("Initialized.*", process)
 285           temp=final.group().split(' ',1)[1]
 286           initialized=" ".join(temp.split())
 287
 288           final= re.search("Ready.*", process)
 289           temp=final.group().split(' ',1)[1]
 290           ready=" ".join(temp.split())
 291
 292           final= re.search("ContainersReady.*", process)
 293           temp=final.group().split(' ',1)[1]
 294           containersready=" ".join(temp.split())
 295
 296           final= re.search("PodScheduled.*", process)
 297           temp=final.group().split(' ',1)[1]
 298           podscheduled=" ".join(temp.split())
 299
 300           if "True"==initialized and "True"==podscheduled and "True"==containersready and "True"==ready:
 301              print("Xapp health status : Healthy")
 302           else:
 303              print("Xapp health status : Unhealthy")
 304              if "True"!=containersready:
 305                print("ContainersReady=False, All the containers in the pod are not ready\n")
 306              elif "True"!=initialized:
 307                print("Initialized=False, Init containers have not yet started\n")
 308              elif "True"!=podscheduled:
 309                print("PodScheduled=False, Pod has not yet scheduled to node\n")
 310              elif "True"!=ready:
 311                print("Ready=False, Pod is not ready to serve any request\n")
 312         except Exception as err:
 313             print(err.output.decode())