xapp_onboarder/xapp_onboarder/helm_controller/xApp_builder.py

   1 ################################################################################
   2 #   Copyright (c) 2020 AT&T Intellectual Property.                             #
   3 #                                                                              #
   4 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   5 #   you may not use this file except in compliance with the License.           #
   6 #   You may obtain a copy of the License at                                    #
   7 #                                                                              #
   8 #       http://www.apache.org/licenses/LICENSE-2.0                             #
   9 #                                                                              #
  10 #   Unless required by applicable law or agreed to in writing, software        #
  11 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  12 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  13 #   See the License for the specific language governing permissions and        #
  14 #   limitations under the License.                                             #
  15 ################################################################################
  16
  17 import logging
  18 import yaml
  19 import json
  20 import os
  21 import io
  22 import subprocess
  23 import shutil
  24 import re
  25 import copy
  26 import platform
  27 import tarfile
  28 import stat
  29 from xapp_onboarder.server import settings
  30 from xapp_onboarder.repo_manager.repo_manager import repo_manager, RepoManagerError
  31 from pkg_resources import resource_filename
  32 from subprocess import PIPE
  33 from xapp_onboarder.repo_manager.repo_manager import requests_retry_session
  34 log = logging.getLogger(__name__)
  35
  36
  37 def indent(text, amount, ch=' '):
  38     padding = amount * ch
  39     return ''.join(padding + line for line in text.splitlines(True))
  40
  41
  42 class xAppError(Exception):
  43     def __init__(self, message, status_code):
  44         # Call the base class constructor with the parameters it needs
  45         super().__init__(message)
  46         self.status_code = status_code
  47
  48
  49 class xApp():
  50     def __init__(self, config_file, schema_file):
  51         self.config_file = config_file
  52         self.schema_file = schema_file
  53
  54         if 'xapp_name' not in self.config_file:
  55             raise xAppError(
  56                 "xApp chart name not found. (Caused by: config-file.json does not contain xapp_name attribute.)", 500)
  57
  58         if 'version' not in self.config_file:
  59             raise xAppError(
  60                 "xApp chart version not found. (Caused by: config-file.json does not contain version attribute.)", 500)
  61
  62         self.chart_name = self.config_file['xapp_name']
  63         self.chart_version = self.config_file['version']
  64         self.configmap_config_json_file = copy.deepcopy(self.config_file)
  65         self.chart_workspace_path = settings.CHART_WORKSPACE_PATH + '/' + self.chart_name + '-' + self.chart_version
  66         if os.path.exists(self.chart_workspace_path):
  67             shutil.rmtree(self.chart_workspace_path)
  68         os.makedirs(self.chart_workspace_path)
  69         shutil.copytree(resource_filename( 'xapp_onboarder', 'resources/xapp-std'), self.chart_workspace_path + '/' + self.chart_name)
  70         self.setup_helm()
  71
  72     def setup_helm(self):
  73         self.helm_client_path = 'helm'
  74         try:
  75             process = subprocess.run([self.helm_client_path], stdout=PIPE, stderr=PIPE, check=True)
  76
  77         except Exception as err:
  78             print(err)
  79             self.download_helm()
  80             self.helm_client_path = settings.CHART_WORKSPACE_PATH + '/helm'
  81
  82     def download_helm(self):
  83         if not os.path.isfile(settings.CHART_WORKSPACE_PATH + '/helm'):
  84             log.info("Helm client missing. Trying to download it.")
  85             helm_file_name = "helm-v{}-{}-amd64.tar.gz".format(settings.HELM_VERSION, platform.system().lower())
  86             helm_download_link = "https://get.helm.sh/" + helm_file_name
  87
  88
  89             try:
  90                 response = requests_retry_session().get(helm_download_link, timeout=settings.HTTP_TIME_OUT)
  91             except Exception as err:
  92                 error_message = "Download helm client failed. (Caused by: " + str(err)+")"
  93                 log.error(error_message)
  94                 raise xAppError(error_message, 500)
  95             else:
  96                 if response.status_code != 200:
  97                     error_message = "Download helm chart failed. Helm repo return status code: "+ str(response.status_code)  +" "+ response.content.decode("utf-8")
  98                     log.error(error_message)
  99                     raise xAppError(error_message, 500)
 100
 101                 file_stream = io.BytesIO(response.content)
 102
 103                 with tarfile.open(fileobj=file_stream) as tar:
 104                     helm_client = tar.extractfile(platform.system().lower() + "-amd64/helm")
 105                     with open(settings.CHART_WORKSPACE_PATH+'/helm', 'wb') as file:
 106                         file.write(helm_client.read())
 107                 st = os.stat(settings.CHART_WORKSPACE_PATH+'/helm')
 108                 os.chmod(settings.CHART_WORKSPACE_PATH+'/helm', st.st_mode | stat.S_IEXEC)
 109
 110
 111
 112
 113     def recursive_convert_config_file(self, node_list=list()):
 114         current_node = self.configmap_config_json_file
 115         helm_value_path = '.Values'
 116         for node in node_list:
 117             current_node = current_node.get(node)
 118             helm_value_path = helm_value_path + '.' + node
 119
 120         if type(current_node) is not dict:
 121             raise TypeError("Recursive write was called on a leaf node.")
 122
 123         for item in current_node.keys():
 124             if type(current_node.get(item)) is not dict:
 125                 current_node[item] = '{{ index '+ helm_value_path +' "'+ item + '" | toJson }}'
 126             else:
 127                 new_node_list = node_list.copy()
 128                 new_node_list.append(item)
 129                 self.recursive_convert_config_file(new_node_list)
 130
 131
 132     def append_config_to_config_map(self):
 133         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/appconfig.yaml', 'a') as outputfile:
 134             self.recursive_convert_config_file()
 135             config_file_json_text = json.dumps(self.configmap_config_json_file, indent=4)
 136             indented_config_text = indent(config_file_json_text, 4)
 137             indented_config_text = re.sub(r"\"{{", '{{', indented_config_text)
 138             indented_config_text = re.sub(r"}}\"", '}}', indented_config_text)
 139             indented_config_text = re.sub(r"\\", '', indented_config_text)
 140             outputfile.write("  config-file.json: |\n")
 141             outputfile.write(indented_config_text)
 142             outputfile.write("\n  schema.json: |\n")
 143             schema_json = json.dumps(self.schema_file, indent=4)
 144             indented_schema_text = indent(schema_json, 4)
 145             outputfile.write(indented_schema_text)
 146
 147
 148 # This is a work around for the bronze release to be backward compatible to the previous xapp standard helm template
 149     def write_config_and_schema(self):
 150         os.makedirs(self.chart_workspace_path + '/' + self.chart_name + '/descriptors')
 151         os.makedirs(self.chart_workspace_path + '/' + self.chart_name + '/config')
 152         with open(self.chart_workspace_path + '/' + self.chart_name + '/descriptors/schema.json', 'w') as outfile:
 153             json.dump(self.schema_file, outfile)
 154         with open(self.chart_workspace_path + '/' + self.chart_name + '/config/config-file.json', 'w') as outfile:
 155             json.dump(self.config_file, outfile)
 156
 157
 158
 159     def add_probes_to_deployment(self):
 160         with open(self.chart_workspace_path + '/' + self.chart_name + '/templates/deployment.yaml', 'a') as outputfile:
 161
 162             for probes in ['readinessProbe', 'livenessProbe']:
 163                 if self.configmap_config_json_file.get(probes):
 164                     probe_definition = self.configmap_config_json_file.get(probes)
 165                     probe_definition_yaml = yaml.dump(probe_definition, width=1000)
 166
 167                     print(probe_definition_yaml)
 168
 169                     indented_probe_definition_yaml = indent(probe_definition_yaml, 12)
 170                     indented_probe_definition_yaml = re.sub(r" \| toJson", '', indented_probe_definition_yaml)
 171                     indented_probe_definition_yaml = re.sub(r"'", '', indented_probe_definition_yaml)
 172                     outputfile.write("          "+probes+":\n")
 173                     outputfile.write(indented_probe_definition_yaml)
 174
 175
 176     def append_config_to_values_yaml(self):
 177         with open(self.chart_workspace_path + '/' + self.chart_name + '/values.yaml', 'a') as outputfile:
 178             yaml.dump(self.config_file, outputfile, default_flow_style=False)
 179
 180
 181     def change_chart_name_version(self):
 182         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'r') as inputfile:
 183             self.chart_yaml = yaml.load(inputfile, Loader=yaml.FullLoader)
 184             self.chart_yaml['version'] = self.chart_version
 185             self.chart_yaml['name'] = self.chart_name
 186
 187         with open(self.chart_workspace_path + '/' + self.chart_name + '/Chart.yaml', 'w') as outputfile:
 188             yaml.dump(self.chart_yaml, outputfile, default_flow_style=False)
 189
 190
 191     def helm_lint(self):
 192         try:
 193             process = subprocess.run([self.helm_client_path, "lint", self.chart_workspace_path + "/" + self.chart_name], stdout=PIPE, stderr=PIPE, check=True)
 194
 195         except OSError as err:
 196             raise xAppError(
 197                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " + str(
 198                     err) + ")", 500)
 199         except subprocess.CalledProcessError as err:
 200             raise xAppError(
 201                 "xApp " + self.chart_name + '-' + self.chart_version + " helm lint failed. (Caused by: " +
 202                 err.stderr.decode("utf-8") +  "\n" + err.stdout.decode("utf-8") + ")", 400)
 203
 204     def package_chart(self):
 205         self.write_config_and_schema()
 206         self.append_config_to_config_map()
 207         self.append_config_to_values_yaml()
 208         self.add_probes_to_deployment()
 209         self.change_chart_name_version()
 210         self.helm_lint()
 211         try:
 212             process = subprocess.run([self.helm_client_path, "package", self.chart_workspace_path + "/" + self.chart_name, "-d"
 213                                ,self.chart_workspace_path,"--save=false"], stdout=PIPE, stderr=PIPE, check=True)
 214
 215         except OSError as err:
 216                 raise xAppError("xApp "+ self.chart_name+'-'+self.chart_version +" packaging failed. (Caused by: "+str(err) +")", 500)
 217         except subprocess.CalledProcessError as err:
 218             raise xAppError(
 219                 "xApp " + self.chart_name + '-' + self.chart_version + " packaging failed. (Caused by: " +
 220                     err.stderr.decode("utf-8") + ")", 500)
 221
 222
 223
 224     def distribute_chart(self):
 225         try:
 226             repo_manager.upload_chart(self)
 227         except RepoManagerError as err:
 228             raise xAppError( "xApp " + self.chart_name + '-' + self.chart_version + " distribution failed. (Caused by: " + str(err) + ")" , err.status_code)
 229