xapp_onboarder/xapp_onboarder/helm_controller/xApp_builder.py

   1 ################################################################################
   2 #   Copyright (c) 2020 AT&T Intellectual Property.                             #
   3 #                                                                              #
   4 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   5 #   you may not use this file except in compliance with the License.           #
   6 #   You may obtain a copy of the License at                                    #
   7 #                                                                              #
   8 #       http://www.apache.org/licenses/LICENSE-2.0                             #
   9 #                                                                              #
  10 #   Unless required by applicable law or agreed to in writing, software        #
  11 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  12 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  13 #   See the License for the specific language governing permissions and        #
  14 #   limitations under the License.                                             #
  15 ################################################################################
  16
  17 import logging
  18 import yaml
  19 import json
  20 import os
  21 import io
  22 import subprocess
  23 import shutil
  24 import re
  25 import copy
  26 import platform
  27 import tarfile
  28 import stat
  29 from xapp_onboarder.server import settings
  30 from xapp_onboarder.repo_manager.repo_manager import repo_manager, RepoManagerError
  31 from pkg_resources import resource_filename
  32 from subprocess import PIPE
  33 from xapp_onboarder.repo_manager.repo_manager import requests_retry_session
  34 log = logging.getLogger(__name__)
  35
  36
  37 def indent(text, amount, ch=' '):
  38     padding = amount * ch
  39     return ''.join(padding + line for line in text.splitlines(True))
  40
  41
  42 class xAppError(Exception):
  43     def __init__(self, message, status_code):
  44         # Call the base class constructor with the parameters it needs
  45         super().__init__(message)
  46         self.status_code = status_code
  47
  48
  49 class xApp():
  50     def __init__(self, config_file, schema_file):
  51         self.config_file = config_file
  52         self.schema_file = schema_file
  53
  54         if 'xapp_name' not in self.config_file:
  55             raise xAppError(
  56                 "xApp chart name not found. (Caused by: config-file.json does not contain xapp_name attribute.)", 500)
  57
  58         if 'version' not in self.config_file:
  59             raise xAppError(
  60                 "xApp chart version not found. (Caused by: config-file.json does not contain version attribute.)", 500)
  61
  62         self.chart_name = self.config_file['xapp_name']
  63         self.chart_version = self.config_file['version']
  64         self.configmap_config_json_file = copy.deepcopy(self.config_file)
  65         self.chart_workspace_path = settings.CHART_WORKSPACE_PATH + '/' + self.chart_name + '-' + self.chart_version
  66         if os.path.exists(self.chart_workspace_path):
  67             shutil.rmtree(self.chart_workspace_path)
  68         os.makedirs(self.chart_workspace_path)
  69         shutil.copytree(resource_filename( 'xapp_onboarder', 'resources/xapp-std'), self.chart_workspace_path + '/' + self.chart_name)
  70         self.setup_helm()
  71
  72     def setup_helm(self):
  73         self.helm_client_path = 'helm'
  74         try:
  75             process = subprocess.run([self.helm_client_path], stdout=PIPE, stderr=PIPE, check=True)
  76
  77         except Exception as err:
  78             print(err)
  79             self.download_helm()
  80             self.helm_client_path = settings.CHART_WORKSPACE_PATH + '/helm'
  81
  82     def download_helm(self):
  83         if not os.path.isfile(settings.CHART_WORKSPACE_PATH + '/helm'):
  84             log.info("Helm client missing. Trying to download it.")
  85             helm_file_name = "helm-v{}-{}-amd64.tar.gz".format(settings.HELM_VERSION, platform.system().lower())
  86             helm_download_link = "https://get.helm.sh/" + helm_file_name
  87
  88
  89             try:
  90                 response = requests_retry_session().get(helm_download_link, timeout=settings.HTTP_TIME_OUT)
  91             except Exception as err:
  92                 error_message = "Download helm client failed. (Caused by: " + str(err)+")"
  93                 log.error(error_message)
  94                 raise xAppError(error_message, 500)
  95             else:
  96                 if response.status_code != 200:
  97                     error_message = "Download helm chart failed. Helm repo return status code: "+ str(response.status_code)  +" "+ response.content.decode("utf-8")
  98                     log.error(error_message)
  99                     raise xAppError(error_message, 500)
 100
 101                 file_stream = io.BytesIO(response.content)
 102
 103                 with tarfile.open(fileobj=file_stream) as tar:
 104                     helm_client = tar.extractfile(platform.system().lower() + "-amd64/helm")
 105                     with open(settings.CHART_WORKSPACE_PATH+'/helm', 'wb') as file:
 106                         file.write(helm_client.read())
 107                 st = os.stat(settings.CHART_WORKSPACE_PATH+'/helm')
 108                 os.chmod(settings.CHART_WORKSPACE_PATH+'/helm', st.st_mode | stat.S_IEXEC)
 109
 110
 111
 112
 113     def recursive_convert_config_file(self, node_list=list()):
 114         current_node = self.configmap_config_json_file
 115         helm_value_path = '.Values'
 116         for node in node_list:
 117             current_node = current_node.get(node)
 118             helm_value_path = helm_value_path + '.' + node
 119
 120         if type(current_node) is not dict:
 121             raise TypeError("Recursive write was called on a leaf node.")
 122
 123         for item in current_node.keys():
 124             if type(current_node.get(item)) is not dict:
 125                 current_node[item] = '{{ '+ helm_value_path +'.'+ item + ' | toJson }}'
 126             else:
 127                 new_node_list = node_list.copy()
 128                 new_node_list.append(item)
 129                 self.recursive_convert_config_file(new_node_list)
 130
 131
 132
 133     def append_config_to_config_map(self):
 134         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/appconfig.yaml', 'a') as outputfile:
 135             self.recursive_convert_config_file()
 136             config_file_json_text = json.dumps(self.configmap_config_json_file, indent=4)
 137             indented_config_text = indent(config_file_json_text, 4)
 138             indented_config_text = re.sub(r"\"{{", '{{', indented_config_text)
 139             indented_config_text = re.sub(r"}}\"", '}}', indented_config_text)
 140             outputfile.write("  config-file.json: |\n")
 141             outputfile.write(indented_config_text)
 142             outputfile.write("\n  schema.json: |\n")
 143             schema_json = json.dumps(self.schema_file, indent=4)
 144             indented_schema_text = indent(schema_json, 4)
 145             outputfile.write(indented_schema_text)
 146
 147
 148 # This is a work around for the bronze release to be backward compatible to the previous xapp standard helm template
 149     def write_config_and_schema(self):
 150         os.makedirs(self.chart_workspace_path + '/' + self.chart_name + '/descriptors')
 151         os.makedirs(self.chart_workspace_path + '/' + self.chart_name + '/config')
 152         with open(self.chart_workspace_path + '/' + self.chart_name + '/descriptors/schema.json', 'w') as outfile:
 153             json.dump(self.schema_file, outfile)
 154         with open(self.chart_workspace_path + '/' + self.chart_name + '/config/config-file.json', 'w') as outfile:
 155             json.dump(self.config_file, outfile)
 156
 157
 158
 159     def add_probes_to_deployment(self):
 160         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/deployment.yaml', 'a') as outputfile:
 161
 162             for probes in ['readinessProbe', 'livenessProbe']:
 163                 if self.configmap_config_json_file.get(probes):
 164                     probe_definition = self.configmap_config_json_file.get(probes)
 165                     probe_definition_yaml = yaml.dump(probe_definition)
 166                     indented_probe_definition_yaml = indent(probe_definition_yaml, 12)
 167                     indented_probe_definition_yaml = re.sub(r" \| toJson", '', indented_probe_definition_yaml)
 168                     indented_probe_definition_yaml = re.sub(r"'", '', indented_probe_definition_yaml)
 169                     outputfile.write("          "+probes+":\n")
 170                     outputfile.write(indented_probe_definition_yaml)
 171
 172
 173     def append_config_to_values_yaml(self):
 174         with open(self.chart_workspace_path + '/' + self.chart_name + '/values.yaml', 'a') as outputfile:
 175             yaml.dump(self.config_file, outputfile, default_flow_style=False)
 176
 177
 178     def change_chart_name_version(self):
 179         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'r') as inputfile:
 180             self.chart_yaml = yaml.load(inputfile, Loader=yaml.FullLoader)
 181             self.chart_yaml['version'] = self.chart_version
 182             self.chart_yaml['name'] = self.chart_name
 183
 184         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'w') as outputfile:
 185             yaml.dump(self.chart_yaml, outputfile, default_flow_style=False)
 186
 187
 188     def helm_lint(self):
 189         try:
 190             process = subprocess.run([self.helm_client_path, "lint", self.chart_workspace_path + "/" + self.chart_name], stdout=PIPE, stderr=PIPE, check=True)
 191
 192         except OSError as err:
 193             raise xAppError(
 194                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " + str(
 195                     err) + ")", 500)
 196         except subprocess.CalledProcessError as err:
 197             raise xAppError(
 198                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " +
 199                 err.stderr.decode("utf-8") +  "\n" + err.stdout.decode("utf-8") + ")", 400)
 200
 201     def package_chart(self):
 202         self.write_config_and_schema()
 203         self.append_config_to_config_map()
 204         self.append_config_to_values_yaml()
 205         self.add_probes_to_deployment()
 206         self.change_chart_name_version()
 207         self.helm_lint()
 208         try:
 209             process = subprocess.run([self.helm_client_path, "package", self.chart_workspace_path + "/" + self.chart_name, "-d"
 210                                ,self.chart_workspace_path,"--save=false"], stdout=PIPE, stderr=PIPE, check=True)
 211
 212         except OSError as err:
 213                 raise xAppError("xApp "+ self.chart_name+'-'+self.chart_version +" packaging failed. (Caused by: "+str(err) +")", 500)
 214         except subprocess.CalledProcessError as err:
 215             raise xAppError(
 216                 "xApp " + self.chart_name + '-' + self.chart_version + " packaging failed. (Caused by: " +
 217                     err.stderr.decode("utf-8") + ")", 500)
 218
 219
 220
 221     def distribute_chart(self):
 222         try:
 223             repo_manager.upload_chart(self)
 224         except RepoManagerError as err:
 225             raise xAppError( "xApp " + self.chart_name + '-' + self.chart_version + " distribution failed. (Caused by: " + str(err) + ")" , err.status_code)
 226