xapp_onboarder/xapp_onboarder/helm_controller/xApp_builder.py

   1 ################################################################################
   2 #   Copyright (c) 2020 AT&T Intellectual Property.                             #
   3 #                                                                              #
   4 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   5 #   you may not use this file except in compliance with the License.           #
   6 #   You may obtain a copy of the License at                                    #
   7 #                                                                              #
   8 #       http://www.apache.org/licenses/LICENSE-2.0                             #
   9 #                                                                              #
  10 #   Unless required by applicable law or agreed to in writing, software        #
  11 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  12 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  13 #   See the License for the specific language governing permissions and        #
  14 #   limitations under the License.                                             #
  15 ################################################################################
  16
  17 import logging
  18 import yaml
  19 import json
  20 import os
  21 import io
  22 import subprocess
  23 import shutil
  24 import re
  25 import copy
  26 import platform
  27 import tarfile
  28 import stat
  29 from xapp_onboarder.server import settings
  30 from xapp_onboarder.repo_manager.repo_manager import repo_manager, RepoManagerError
  31 from pkg_resources import resource_filename
  32 from subprocess import PIPE
  33 from xapp_onboarder.repo_manager.repo_manager import requests_retry_session
  34 log = logging.getLogger(__name__)
  35
  36
  37 def indent(text, amount, ch=' '):
  38     padding = amount * ch
  39     return ''.join(padding + line for line in text.splitlines(True))
  40
  41
  42 class xAppError(Exception):
  43     def __init__(self, message, status_code):
  44         # Call the base class constructor with the parameters it needs
  45         super().__init__(message)
  46         self.status_code = status_code
  47
  48
  49 class xApp():
  50     def __init__(self, config_file, schema_file):
  51         self.config_file = config_file
  52         self.schema_file = schema_file
  53
  54         if 'xapp_name' not in self.config_file:
  55             raise xAppError(
  56                 "xApp chart name not found. (Caused by: config-file.json does not contain xapp_name attribute.)", 500)
  57
  58         if 'version' not in self.config_file:
  59             raise xAppError(
  60                 "xApp chart version not found. (Caused by: config-file.json does not contain version attribute.)", 500)
  61
  62         self.chart_name = self.config_file['xapp_name']
  63         self.chart_version = self.config_file['version']
  64         self.configmap_config_json_file = copy.deepcopy(self.config_file)
  65         self.chart_workspace_path = settings.CHART_WORKSPACE_PATH + '/' + self.chart_name + '-' + self.chart_version
  66         if os.path.exists(self.chart_workspace_path):
  67             shutil.rmtree(self.chart_workspace_path)
  68         os.makedirs(self.chart_workspace_path)
  69         shutil.copytree(resource_filename( 'xapp_onboarder', 'resources/xapp-std'), self.chart_workspace_path + '/' + self.chart_name)
  70         self.helm_client_path = settings.CHART_WORKSPACE_PATH + '/helm'
  71         self.setup_helm()
  72
  73     def setup_helm(self):
  74         if not os.path.isfile(settings.CHART_WORKSPACE_PATH + '/helm'):
  75             log.info("Helm client missing. Trying to download it.")
  76             helm_file_name = "helm-v{}-{}-amd64.tar.gz".format(settings.HELM_VERSION, platform.system().lower())
  77             helm_download_link = "https://get.helm.sh/" + helm_file_name
  78
  79
  80             try:
  81                 response = requests_retry_session().get(helm_download_link, timeout=settings.HTTP_TIME_OUT)
  82             except Exception as err:
  83                 error_message = "Download helm client failed. (Caused by: " + str(err)+")"
  84                 log.error(error_message)
  85                 raise xAppError(error_message, 500)
  86             else:
  87                 if response.status_code != 200:
  88                     error_message = "Download helm chart failed. Helm repo return status code: "+ str(response.status_code)  +" "+ response.content.decode("utf-8")
  89                     log.error(error_message)
  90                     raise xAppError(error_message, 500)
  91
  92                 file_stream = io.BytesIO(response.content)
  93
  94                 with tarfile.open(fileobj=file_stream) as tar:
  95                     helm_client = tar.extractfile(platform.system().lower() + "-amd64/helm")
  96                     with open(settings.CHART_WORKSPACE_PATH+'/helm', 'wb') as file:
  97                         file.write(helm_client.read())
  98                 st = os.stat(settings.CHART_WORKSPACE_PATH+'/helm')
  99                 os.chmod(settings.CHART_WORKSPACE_PATH+'/helm', st.st_mode | stat.S_IEXEC)
 100
 101
 102
 103
 104     def recursive_convert_config_file(self, node_list=list()):
 105         current_node = self.configmap_config_json_file
 106         helm_value_path = '.Values'
 107         for node in node_list:
 108             current_node = current_node.get(node)
 109             helm_value_path = helm_value_path + '.' + node
 110
 111         if type(current_node) is not dict:
 112             raise TypeError("Recursive write was called on a leaf node.")
 113
 114         for item in current_node.keys():
 115             if type(current_node.get(item)) is not dict:
 116                 current_node[item] = '{{ '+ helm_value_path +'.'+ item + ' | toJson }}'
 117             else:
 118                 new_node_list = node_list.copy()
 119                 new_node_list.append(item)
 120                 self.recursive_convert_config_file(new_node_list)
 121
 122
 123
 124     def append_config_to_config_map(self):
 125         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/appconfig.yaml', 'a') as outputfile:
 126             self.recursive_convert_config_file()
 127             config_file_json_text = json.dumps(self.configmap_config_json_file, indent=4)
 128             indented_config_text = indent(config_file_json_text, 4)
 129             indented_config_text = re.sub(r"\"{{", '{{', indented_config_text)
 130             indented_config_text = re.sub(r"}}\"", '}}', indented_config_text)
 131             outputfile.write("  config-file.json: |\n")
 132             outputfile.write(indented_config_text)
 133             outputfile.write("\n  schema.json: |\n")
 134             schema_json = json.dumps(self.schema_file, indent=4)
 135             indented_schema_text = indent(schema_json, 4)
 136             outputfile.write(indented_schema_text)
 137
 138
 139     def add_probes_to_deployment(self):
 140         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/deployment.yaml', 'a') as outputfile:
 141
 142             for probes in ['readinessProbe', 'livenessProbe']:
 143                 if self.configmap_config_json_file.get(probes):
 144                     probe_definition = self.configmap_config_json_file.get(probes)
 145                     probe_definition_yaml = yaml.dump(probe_definition)
 146                     indented_probe_definition_yaml = indent(probe_definition_yaml, 12)
 147                     indented_probe_definition_yaml = re.sub(r" \| toJson", '', indented_probe_definition_yaml)
 148                     indented_probe_definition_yaml = re.sub(r"'", '', indented_probe_definition_yaml)
 149                     outputfile.write("          "+probes+":\n")
 150                     outputfile.write(indented_probe_definition_yaml)
 151
 152
 153
 154     def append_config_to_values_yaml(self):
 155         with open(self.chart_workspace_path + '/' + self.chart_name + '/values.yaml', 'a') as outputfile:
 156             yaml.dump(self.config_file, outputfile, default_flow_style=False)
 157
 158
 159     def append_env_to_config_map(self):
 160         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/appenv.yaml', 'a') as outputfile:
 161             append = {}
 162             if settings.DBAAS_MASTER_NAME:
 163                 master_name = settings.DBAAS_MASTER_NAME
 164                 service_host = settings.DBAAS_SERVICE_HOST
 165                 sentinel_port = settings.DBAAS_SERVICE_SENTINEL_PORT
 166                 if not service_host:
 167                     raise xAppError(
 168                         "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_HOST'. (Caused by: Misconfiguration of temp deployment)", 500)
 169                 if not sentinel_port:
 170                     raise xAppError(
 171                         "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_SENTINEL_PORT'. (Caused by: Misconfiguration of temp deployment)", 500)
 172
 173                 append['DBAAS_MASTER_NAME'] = master_name
 174                 append['DBAAS_SERVICE_HOST'] = service_host
 175                 append['DBAAS_SERVICE_SENTINEL_PORT'] = sentinel_port
 176             elif settings.DBAAS_SERVICE_HOST:
 177                 service_host = settings.DBAAS_SERVICE_HOST
 178                 service_port = settings.DBAAS_SERVICE_PORT
 179                 if not service_port:
 180                     raise xAppError(
 181                         "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_PORT'. (Caused by: Misconfiguration of temp deployment)", 500)
 182                 append['DBAAS_SERVICE_HOST'] = service_host
 183                 append['DBAAS_SERVICE_PORT'] = service_port
 184             else:
 185                 raise xAppError(
 186                     "Internal failure. Cannot find environment variable 'DBAAS_SERVICE_HOST' or 'DBAAS_MASTER_NAME'. (Caused by: Misconfiguration of temp deployment)",
 187                     500)
 188             output_yaml = yaml.dump(append)
 189             indented_output_yaml = indent(output_yaml, 2)
 190             outputfile.write(indented_output_yaml)
 191
 192
 193     def change_chart_name_version(self):
 194         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'r') as inputfile:
 195             self.chart_yaml = yaml.load(inputfile, Loader=yaml.FullLoader)
 196             self.chart_yaml['version'] = self.chart_version
 197             self.chart_yaml['name'] = self.chart_name
 198
 199         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'w') as outputfile:
 200             yaml.dump(self.chart_yaml, outputfile, default_flow_style=False)
 201
 202
 203     def helm_lint(self):
 204         try:
 205             process = subprocess.run([self.helm_client_path, "lint", self.chart_workspace_path + "/" + self.chart_name], stdout=PIPE, stderr=PIPE, check=True)
 206
 207         except OSError as err:
 208             raise xAppError(
 209                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " + str(
 210                     err) + ")", 500)
 211         except subprocess.CalledProcessError as err:
 212             raise xAppError(
 213                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " +
 214                 err.stderr.decode("utf-8") +  "\n" + err.stdout.decode("utf-8") + ")", 400)
 215
 216     def package_chart(self):
 217         self.append_config_to_config_map()
 218         self.append_config_to_values_yaml()
 219         self.append_env_to_config_map()
 220         self.add_probes_to_deployment()
 221         self.change_chart_name_version()
 222         self.helm_lint()
 223         try:
 224             process = subprocess.run([self.helm_client_path, "package", self.chart_workspace_path + "/" + self.chart_name, "-d"
 225                                ,self.chart_workspace_path,"--save=false"], stdout=PIPE, stderr=PIPE, check=True)
 226
 227         except OSError as err:
 228                 raise xAppError("xApp "+ self.chart_name+'-'+self.chart_version +" packaging failed. (Caused by: "+str(err) +")", 500)
 229         except subprocess.CalledProcessError as err:
 230             raise xAppError(
 231                 "xApp " + self.chart_name + '-' + self.chart_version + " packaging failed. (Caused by: " +
 232                     err.stderr.decode("utf-8") + ")", 500)
 233
 234
 235
 236     def distribute_chart(self):
 237         try:
 238             repo_manager.upload_chart(self)
 239         except RepoManagerError as err:
 240             raise xAppError( "xApp " + self.chart_name + '-' + self.chart_version + " distribution failed. (Caused by: " + str(err) + ")" , err.status_code)
 241