xapp_onboarder/xapp_onboarder/helm_controller/xApp_builder.py

   1 ################################################################################
   2 #   Copyright (c) 2020 AT&T Intellectual Property.                             #
   3 #                                                                              #
   4 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   5 #   you may not use this file except in compliance with the License.           #
   6 #   You may obtain a copy of the License at                                    #
   7 #                                                                              #
   8 #       http://www.apache.org/licenses/LICENSE-2.0                             #
   9 #                                                                              #
  10 #   Unless required by applicable law or agreed to in writing, software        #
  11 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  12 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  13 #   See the License for the specific language governing permissions and        #
  14 #   limitations under the License.                                             #
  15 ################################################################################
  16
  17 import logging
  18 import yaml
  19 import json
  20 import os
  21 import io
  22 import subprocess
  23 import shutil
  24 import re
  25 import copy
  26 import platform
  27 import tarfile
  28 import stat
  29 from xapp_onboarder.server import settings
  30 from xapp_onboarder.repo_manager.repo_manager import repo_manager, RepoManagerError
  31 from pkg_resources import resource_filename
  32 from subprocess import PIPE
  33 from xapp_onboarder.repo_manager.repo_manager import requests_retry_session
  34 log = logging.getLogger(__name__)
  35
  36
  37 def indent(text, amount, ch=' '):
  38     padding = amount * ch
  39     return ''.join(padding + line for line in text.splitlines(True))
  40
  41
  42 class xAppError(Exception):
  43     def __init__(self, message, status_code):
  44         # Call the base class constructor with the parameters it needs
  45         super().__init__(message)
  46         self.status_code = status_code
  47
  48
  49 class xApp():
  50     def __init__(self, config_file, schema_file):
  51         self.config_file = config_file
  52         self.schema_file = schema_file
  53
  54         if 'xapp_name' not in self.config_file:
  55             raise xAppError(
  56                 "xApp chart name not found. (Caused by: config-file.json does not contain xapp_name attribute.)", 500)
  57
  58         if 'version' not in self.config_file:
  59             raise xAppError(
  60                 "xApp chart version not found. (Caused by: config-file.json does not contain version attribute.)", 500)
  61
  62         self.chart_name = self.config_file['xapp_name']
  63         self.chart_version = self.config_file['version']
  64         self.configmap_config_json_file = copy.deepcopy(self.config_file)
  65         self.chart_workspace_path = settings.CHART_WORKSPACE_PATH + '/' + self.chart_name + '-' + self.chart_version
  66         if os.path.exists(self.chart_workspace_path):
  67             shutil.rmtree(self.chart_workspace_path)
  68         os.makedirs(self.chart_workspace_path)
  69         shutil.copytree(resource_filename( 'xapp_onboarder', 'resources/xapp-std'), self.chart_workspace_path + '/' + self.chart_name)
  70         self.setup_helm()
  71
  72     def setup_helm(self):
  73         self.helm_client_path = 'helm'
  74         try:
  75             process = subprocess.run([self.helm_client_path], stdout=PIPE, stderr=PIPE, check=True)
  76
  77         except Exception as err:
  78             print(err)
  79             self.download_helm()
  80             self.helm_client_path = settings.CHART_WORKSPACE_PATH + '/helm'
  81
  82     def download_helm(self):
  83         if not os.path.isfile(settings.CHART_WORKSPACE_PATH + '/helm'):
  84             log.info("Helm client missing. Trying to download it.")
  85             helm_file_name = "helm-v{}-{}-amd64.tar.gz".format(settings.HELM_VERSION, platform.system().lower())
  86             helm_download_link = "https://get.helm.sh/" + helm_file_name
  87
  88
  89             try:
  90                 response = requests_retry_session().get(helm_download_link, timeout=settings.HTTP_TIME_OUT)
  91             except Exception as err:
  92                 error_message = "Download helm client failed. (Caused by: " + str(err)+")"
  93                 log.error(error_message)
  94                 raise xAppError(error_message, 500)
  95             else:
  96                 if response.status_code != 200:
  97                     error_message = "Download helm chart failed. Helm repo return status code: "+ str(response.status_code)  +" "+ response.content.decode("utf-8")
  98                     log.error(error_message)
  99                     raise xAppError(error_message, 500)
 100
 101                 file_stream = io.BytesIO(response.content)
 102
 103                 with tarfile.open(fileobj=file_stream) as tar:
 104                     helm_client = tar.extractfile(platform.system().lower() + "-amd64/helm")
 105                     with open(settings.CHART_WORKSPACE_PATH+'/helm', 'wb') as file:
 106                         file.write(helm_client.read())
 107                 st = os.stat(settings.CHART_WORKSPACE_PATH+'/helm')
 108                 os.chmod(settings.CHART_WORKSPACE_PATH+'/helm', st.st_mode | stat.S_IEXEC)
 109
 110
 111
 112
 113     def recursive_convert_config_file(self, node_list=list()):
 114         current_node = self.configmap_config_json_file
 115         helm_value_path = '.Values'
 116         for node in node_list:
 117             current_node = current_node.get(node)
 118             helm_value_path = helm_value_path + '.' + node
 119
 120         if type(current_node) is not dict:
 121             raise TypeError("Recursive write was called on a leaf node.")
 122
 123         for item in current_node.keys():
 124             if type(current_node.get(item)) is not dict:
 125                 current_node[item] = '{{ '+ helm_value_path +'.'+ item + ' | toJson }}'
 126             else:
 127                 new_node_list = node_list.copy()
 128                 new_node_list.append(item)
 129                 self.recursive_convert_config_file(new_node_list)
 130
 131
 132
 133     def append_config_to_config_map(self):
 134         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/appconfig.yaml', 'a') as outputfile:
 135             self.recursive_convert_config_file()
 136             config_file_json_text = json.dumps(self.configmap_config_json_file, indent=4)
 137             indented_config_text = indent(config_file_json_text, 4)
 138             indented_config_text = re.sub(r"\"{{", '{{', indented_config_text)
 139             indented_config_text = re.sub(r"}}\"", '}}', indented_config_text)
 140             outputfile.write("  config-file.json: |\n")
 141             outputfile.write(indented_config_text)
 142             outputfile.write("\n  schema.json: |\n")
 143             schema_json = json.dumps(self.schema_file, indent=4)
 144             indented_schema_text = indent(schema_json, 4)
 145             outputfile.write(indented_schema_text)
 146
 147
 148     def add_probes_to_deployment(self):
 149         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/deployment.yaml', 'a') as outputfile:
 150
 151             for probes in ['readinessProbe', 'livenessProbe']:
 152                 if self.configmap_config_json_file.get(probes):
 153                     probe_definition = self.configmap_config_json_file.get(probes)
 154                     probe_definition_yaml = yaml.dump(probe_definition)
 155                     indented_probe_definition_yaml = indent(probe_definition_yaml, 12)
 156                     indented_probe_definition_yaml = re.sub(r" \| toJson", '', indented_probe_definition_yaml)
 157                     indented_probe_definition_yaml = re.sub(r"'", '', indented_probe_definition_yaml)
 158                     outputfile.write("          "+probes+":\n")
 159                     outputfile.write(indented_probe_definition_yaml)
 160
 161
 162
 163     def append_config_to_values_yaml(self):
 164         with open(self.chart_workspace_path + '/' + self.chart_name + '/values.yaml', 'a') as outputfile:
 165             yaml.dump(self.config_file, outputfile, default_flow_style=False)
 166
 167
 168     def append_env_to_config_map(self):
 169         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/appenv.yaml', 'a') as outputfile:
 170             append = {}
 171             if settings.DBAAS_MASTER_NAME:
 172                 master_name = settings.DBAAS_MASTER_NAME
 173                 service_host = settings.DBAAS_SERVICE_HOST
 174                 sentinel_port = settings.DBAAS_SERVICE_SENTINEL_PORT
 175                 if not service_host:
 176                     raise xAppError(
 177                         "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_HOST'. (Caused by: Misconfiguration of temp deployment)", 500)
 178                 if not sentinel_port:
 179                     raise xAppError(
 180                         "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_SENTINEL_PORT'. (Caused by: Misconfiguration of temp deployment)", 500)
 181
 182                 append['DBAAS_MASTER_NAME'] = master_name
 183                 append['DBAAS_SERVICE_HOST'] = service_host
 184                 append['DBAAS_SERVICE_SENTINEL_PORT'] = sentinel_port
 185             elif settings.DBAAS_SERVICE_HOST:
 186                 service_host = settings.DBAAS_SERVICE_HOST
 187                 service_port = settings.DBAAS_SERVICE_PORT
 188                 if not service_port:
 189                     raise xAppError(
 190                         "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_PORT'. (Caused by: Misconfiguration of temp deployment)", 500)
 191                 append['DBAAS_SERVICE_HOST'] = service_host
 192                 append['DBAAS_SERVICE_PORT'] = service_port
 193             else:
 194                 raise xAppError(
 195                     "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_HOST' or 'DBAAS_MASTER_NAME'. (Caused by: Misconfiguration of temp deployment)",
 196                     500)
 197             output_yaml = yaml.dump(append)
 198             indented_output_yaml = indent(output_yaml, 2)
 199             outputfile.write(indented_output_yaml)
 200
 201
 202     def change_chart_name_version(self):
 203         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'r') as inputfile:
 204             self.chart_yaml = yaml.load(inputfile, Loader=yaml.FullLoader)
 205             self.chart_yaml['version'] = self.chart_version
 206             self.chart_yaml['name'] = self.chart_name
 207
 208         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'w') as outputfile:
 209             yaml.dump(self.chart_yaml, outputfile, default_flow_style=False)
 210
 211
 212     def helm_lint(self):
 213         try:
 214             process = subprocess.run([self.helm_client_path, "lint", self.chart_workspace_path + "/" + self.chart_name], stdout=PIPE, stderr=PIPE, check=True)
 215
 216         except OSError as err:
 217             raise xAppError(
 218                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " + str(
 219                     err) + ")", 500)
 220         except subprocess.CalledProcessError as err:
 221             raise xAppError(
 222                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " +
 223                 err.stderr.decode("utf-8") +  "\n" + err.stdout.decode("utf-8") + ")", 400)
 224
 225     def package_chart(self):
 226         self.append_config_to_config_map()
 227         self.append_config_to_values_yaml()
 228         self.append_env_to_config_map()
 229         self.add_probes_to_deployment()
 230         self.change_chart_name_version()
 231         self.helm_lint()
 232         try:
 233             process = subprocess.run([self.helm_client_path, "package", self.chart_workspace_path + "/" + self.chart_name, "-d"
 234                                ,self.chart_workspace_path,"--save=false"], stdout=PIPE, stderr=PIPE, check=True)
 235
 236         except OSError as err:
 237                 raise xAppError("xApp "+ self.chart_name+'-'+self.chart_version +" packaging failed. (Caused by: "+str(err) +")", 500)
 238         except subprocess.CalledProcessError as err:
 239             raise xAppError(
 240                 "xApp " + self.chart_name + '-' + self.chart_version + " packaging failed. (Caused by: " +
 241                     err.stderr.decode("utf-8") + ")", 500)
 242
 243
 244
 245     def distribute_chart(self):
 246         try:
 247             repo_manager.upload_chart(self)
 248         except RepoManagerError as err:
 249             raise xAppError( "xApp " + self.chart_name + '-' + self.chart_version + " distribution failed. (Caused by: " + str(err) + ")" , err.status_code)
 250