From: E. Scott Daniels <daniels@research.att.com>
Date: Mon, 1 Feb 2021 16:51:19 +0000 (-0500)
Subject: Add new xapp manager interface
X-Git-Tag: 1.9.0~1
X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=3d4b88135c1a1429d8cecb7380cfc2f8c67d04ef;p=ric-app%2Fmc.git

Add new xapp manager interface

The new interface for xAPP manager requires that the xAPP send
a "regestration" when the application/container starts, and an
unregister when shutting down.  This change adds a new "xam"
(xappManager) script which is invoked by the container start
script to do this.  There are also a couple of other utilities
which make bundling the confi data for the curl call easier.

Issue-ID: RIC-739

Signed-off-by: E. Scott Daniels <daniels@research.att.com>
Change-Id: I5eb4173e6bce7e7aca39d7afac36a9fc949d9a47
Signed-off-by: E. Scott Daniels <daniels@research.att.com>
---

diff --git a/mc-core/Dockerfile b/mc-core/Dockerfile
index 0f6fa93..56b4765 100644
--- a/mc-core/Dockerfile
+++ b/mc-core/Dockerfile
@@ -16,11 +16,11 @@
 
 ARG STAGE_DIR=/mc
 
-FROM nexus3.o-ran-sc.org:10004/o-ran-sc/bldr-ubuntu18-c-go:9-u18.04 AS project-build
+FROM nexus3.o-ran-sc.org:10004/o-ran-sc/bldr-ubuntu18-c-go:1.9.0 AS project-build
 
 ARG STAGE_DIR
 
-ARG RMR_VER=4.4.6
+ARG RMR_VER=4.5.2
 
 RUN wget -nv --content-disposition https://packagecloud.io/o-ran-sc/release/packages/debian/stretch/rmr_${RMR_VER}_amd64.deb/download.deb
 RUN wget -nv --content-disposition https://packagecloud.io/o-ran-sc/release/packages/debian/stretch/rmr-dev_${RMR_VER}_amd64.deb/download.deb
@@ -78,6 +78,11 @@ RUN python generate_runall.py
 # now install the binaries and libraries into smaller docker image
 FROM nexus3.o-ran-sc.org:10002/o-ran-sc/ric-app-mc-listener:1.7.0
 
+# keep this close to the top to prevent lengthy rebuilds during testing
+RUN apt-get update && \
+    apt-get install -y curl python python-pip libboost-all-dev libhiredis-dev && \
+    apt-get clean
+
 ARG STAGE_DIR
 
 COPY --from=project-build ${STAGE_DIR}/gs-lite/demo/queries /mc/gs-lite/demo/queries
@@ -92,10 +97,7 @@ WORKDIR /opt/ric/config
 COPY --from=project-build ${STAGE_DIR}/mc_deployment.json /opt/ric/config/config-file.json
 
 COPY container_start.sh /playpen/bin/
-
-RUN apt-get update && \
-    apt-get install -y curl python python-pip libboost-all-dev libhiredis-dev && \
-    apt-get clean
+COPY package/*.py package/*.sh /playpen/bin/
 
 RUN ldconfig
 RUN pip install protobuf
diff --git a/mc-core/container-tag.yaml b/mc-core/container-tag.yaml
index 8b6a3d0..35ff9fb 100644
--- a/mc-core/container-tag.yaml
+++ b/mc-core/container-tag.yaml
@@ -1,4 +1,4 @@
 ---
-tag: '1.0.9'
+tag: '1.0.10'
 
 # this is used by the CI jobs to tag the image it builds
diff --git a/mc-core/container_start.sh b/mc-core/container_start.sh
index b241ecd..40008f9 100755
--- a/mc-core/container_start.sh
+++ b/mc-core/container_start.sh
@@ -25,20 +25,29 @@
 #			This starts all of the related processes which normally would
 #			be started in individual containers.
 #
-#			There are two environment variables which affect the operation
+#			There are environment variables which affect the operation
 #			of this script:
-#				USE_NNG -- if set to !0 then the NNG version of the listener
-#							is started; undefined or when 0 then the SI95
-#							version is used.
 #
-#				GSLITE_ROOT -- Assumed to be the root directory for the 
+#				GSLITE_ROOT -- Assumed to be the root directory for the
 #							core MC xAPP. If not defined, /mc/gs-lite is
 #							assumed.
 #
+#			When NOT running in simulation mode, a registration message is
+#			sent to the xAPP manager via the registration script in /playpen.
+#			An unregister message is "queued" and should be sent when this
+#			script receives a terminating event, or exits normally.
+#
 # Date:		13 February 2019
 # Author:	E. Scott Daniels
 # ----------------------------------------------------------------------
 
+# MUST have a posix style function declaration!
+unreg() {
+	trap - EXIT						# prevent running this again when we force the exit
+	/playpen/bin/xam_register -U
+	exit
+}
+
 set -e
 
 FIFO_DIR="/tmp/mcl/fifos"
@@ -50,20 +59,22 @@ mkdir -p $FIFO_DIR
 
 if [ "$SIMULATOR_MODE" != "true" ]
 then
-# --- start "sidecars" first. They are expected to need /playpen as the working dir
+	# --- start "sidecars" first. They are expected to need /playpen as the working dir
 
-(
-	cd /playpen
-	if [ "$RMR_PORT" != "" ]
-	then
-		bin/mc_listener -p $RMR_PORT
-	else
-		bin/mc_listener
-	fi
-) >/tmp/listener.std 2>&1 &
+	(
+		cd /playpen
+		if [ "$RMR_PORT" != "" ]
+		then
+			bin/mc_listener -p $RMR_PORT
+		else
+			bin/mc_listener
+		fi
+	) >/tmp/listener.std 2>&1 &
 
-echo "listener was started" >&2
+	echo "listener was started" >&2
 
+	trap 'unreg' EXIT 1 2 3 4 15				# unregister on exit/hup/quit/term
+	/playpen/bin/xam_register.sh				# register the xapp now that listener is up
 fi
 
 
@@ -71,4 +82,3 @@ fi
 cd ${GSLITE_ROOT:-/mc/gs-lite}/demo/queries
 ./runall
 
-
diff --git a/mc-core/package/README b/mc-core/package/README
new file mode 100644
index 0000000..4768666
--- /dev/null
+++ b/mc-core/package/README
@@ -0,0 +1,17 @@
+
+This directory contains "package" level scripts which are needed either
+for building the MC image, or are needed when starting or stopping
+the image in a container. These are not really a part of the xAPP.
+
+
+encode_json.py	-- reads a json file and generates the quoted output
+	suitable for passing on a curl command etc.
+
+j2src.py  -- parses the config file (descriptor file) to extract info
+	needed at the time the start script is run.
+
+xam_register.sh -- sends a register or unregister request to the xAPP
+	manager process.
+
+
+These scripts should all be copied to /playpen/bin in the image.
diff --git a/mc-core/package/encode_json.py b/mc-core/package/encode_json.py
new file mode 100755
index 0000000..c59d595
--- /dev/null
+++ b/mc-core/package/encode_json.py
@@ -0,0 +1,34 @@
+#! /usr/bin/env python3
+#----------------------------------------------------------------------------------
+#
+#   Copyright (c) 2021 AT&T Intellectual Property.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#---------------------------------------------------------------------------------
+
+
+#   Abstract:   This will read a json file and generate it as an escaped string on
+#               standard out.  Simple way of stuffing the config into a curl command.
+#
+#   Date:       28 January 2021
+#   Author:     E. Scott Daniels
+# ---------------------------------------------------------------------------------
+
+import json
+import sys
+
+f = open( sys.argv[1] )
+j = json.load( f )
+qj = json.dumps( j )
+print( qj.replace( '"', '\\"' ) )
diff --git a/mc-core/package/j2src.py b/mc-core/package/j2src.py
new file mode 100755
index 0000000..22ecb04
--- /dev/null
+++ b/mc-core/package/j2src.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+#   vi: et ts=4 sw=4 :
+
+#----------------------------------------------------------------------------------
+#
+#   Copyright (c) 2021 AT&T Intellectual Property.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#---------------------------------------------------------------------------------
+
+
+#   Abstract:   A config file (json) parser that looks for one or more "descriptions" and outputs
+#               shell styled variable assignments that can be sourced by a script. Descriptions
+#               are of the form:
+#                   <field>:[<field>:...:{*|shell_var_name}
+#
+#               The description "xapp_name:*"  will find the field xapp_name at the top level
+#               and generate the variable assignment using the field name. The description
+#               "xapp_name:xname" would find the same field but generate "xname=value" in the
+#               output.
+#
+#               It may be necssary to pull a field from an array of objects where a second field
+#               in the object has a desired value.  As an example, in the messaging section there
+#               is an expected array of ports with each port having a name. To exctact a field
+#               like this, the final field in the list may have the form:
+#                   <name>[]<match_name>=<desired-value>@<field-name>
+#
+#               For   "messaging:port[]name=rmr-data@port"
+#               The messaging object is first located, and each element in the port array is examined.
+#               when the element which contains the field "name:, with a value of "rmr-data",
+#               the value of "port" is assigned to the output shell variable name.
+#
+#               Limitations:   This only allows one array to be traversed, and it is assumed to be the
+#               last field.  In other words, nested object arrays are not supported.
+#
+#               Usage:      j2src <config-file-name> <description> [<description>...]
+#
+#   Date:       29 Janurary 2021
+#   Author:     E. Scott Daniels
+# ------------------------------------------------------------------------------------------------------
+import sys
+import json
+
+#   Parse the description (see above) and return name and value to the caller. None,None is returned
+#   when we have an error, or cannot find the field. Debug strings MUST start with # so that they don't
+#   affect shell parsing of output.
+#
+def parse( pj, description, debug=False ) :
+    tokens = description.split( ":" )           # split fields, last is the output name or *
+    out_name = tokens[-1]
+    value = None
+
+    if len( tokens ) < 2 :
+        print( "## ERR ## badly formed description: %s" % description )
+        return None, None
+
+    for i in range( len( tokens ) - 1 )  :
+        atoks = tokens[i].split( "[]" )
+        if len( atoks ) > 1 :                            # array;  [0] is the name, [1] is name=value@desired-name
+            if atoks[0] in pj  :
+                nv = atoks[1].split( "=" )
+                name = nv[0]
+                sv = nv[1].split( "@" )
+                if len( sv ) < 2 :
+                    if( debug ) :
+                        print( "## ERR ## badly formed capture string: missing 'value<desired'" )
+                    return None, None
+
+                match_val = sv[0]
+                pull_name = sv[1]
+                if out_name == "*" :
+                    out_name = pull_name
+
+                ao = pj[atoks[0]]                       # directly at the array object
+                for i in range( len( ao ) ) :           # run each element
+                    if name in ao[i] :
+                        if ao[i][name] == match_val :       # this is the one we want
+                            if pull_name in ao[i] :
+                                return out_name, str( ao[i][pull_name] )            # all things go back as string
+
+                    if debug :
+                        print( "## WRN ## field is not in, or match value %s does NOT match, in %s[%d]: %s" % ( match_val, atoks[0], i, ao[i][name] ) )
+
+                return None, None               # nothing matched and returned; bail now; array must be last field
+
+            else :
+                if debug :
+                    print( "## WRN ## array %s is not found in %s" % (atoks[0], tokens[i]) )
+                return None, None
+        else :
+            if atoks[0] in pj :
+                pj = pj[atoks[0]]           # if not last, this will be an object (we hope)
+                name = atoks[0]
+                value = pj                  # last one should just be a field to yank
+            else :
+                if debug :
+                    print( "## WRN ## field not found: %s" % atoks[0] )
+                return None, None
+
+
+    if out_name == "*" :
+        return name, value
+    return out_name, value
+
+
+# take the name, value and print it as a valid shell variable. We convert True/False or true/false to
+# 1 and 0.
+#
+def print_svar( vname, value ) :
+        if value == "True" or value == "true" :
+            value = "1"
+        if value == "False" or value == "false" :
+            value = "0"
+        if not value.isnumeric() :
+            value = '"' + value + '"'
+
+        vname = vname.replace( " ", "_" )       # ensure it's a valid shell variable name
+        vname = vname.replace( "-", "_" )
+
+        print( "%s=%s" % ( vname, value ) )
+
+
+
+# --------------------------------------------------------------------------------------------------------------
+
+aidx = 1
+debug = False
+if sys.argv[1] == "debug" :
+    aidx += 1
+    debug = True
+
+f = open( sys.argv[aidx] )
+pj = json.load( f )
+f.close()
+aidx += 1
+
+for i in range( aidx, len( sys.argv ) ) :
+    name, val = parse( pj, sys.argv[i], debug )
+
+    if name != None  and  val != None :
+        print_svar( name, val )
diff --git a/mc-core/package/xam_register.sh b/mc-core/package/xam_register.sh
new file mode 100755
index 0000000..a553cd3
--- /dev/null
+++ b/mc-core/package/xam_register.sh
@@ -0,0 +1,192 @@
+# vim: noet ts=4 :
+#----------------------------------------------------------------------------------
+#
+#   Copyright (c) 2021 AT&T Intellectual Property.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#---------------------------------------------------------------------------------
+
+# --------------------------------------------------------------------------------------------------------------
+#	Abstract:	This will send an application registration message to
+#				the xAPP manager.
+#
+#				A note about the field descriptions given to j2src programme:
+#					They define what we need from the config file. Input to j2src is
+#
+#						field:[field:...]:{*|var-name}
+#
+#					fields are considered to be an "object" if not the last. The last field may be
+#					of  the form  name[]match=value@pull-name
+#					where:
+#						the named field is an array of objects and "match" is a field in each
+#						object. Each element is tested to see if the match field's value is "value"
+#						and when true, the "pull-name" field's value is used.  The description is
+#						used in our list to pull the rmr-data port from the messageing section.
+#
+#				Notes about URLs and DNS names:
+#					According to those in the know, the xAPP manager being a platform component will have
+#					a fixed DNS name, and thus we expect that.
+#
+#					In the less than obvious world of kubernetes we cannot depend on hostname actually being
+#					the DNS name for our stuff. So, we assume that if the DN name is NOT the host name, the
+#					variable will be set to the DNS name as RMR also depends on this for things.
+#					If it's not set, we'll default to hostname.
+#
+#	Date:		28 January 2021
+#	Author:		E. Scott Daniels
+# --------------------------------------------------------------------------------------------------------------
+
+# quote the two parms passed in making json style output quoting less cumbersome inline.
+function quote {
+	echo "\"$1\": \"$2\""
+}
+
+# ------------------------------------------------------------------------------------------------
+xam_url="http://service-ricplt-appmgr-http:8080/ric/v1"		# where we expect xAPP mgr to be waiting
+svc_name="${RMR_SRC_ID:-$( hostname )}"						# the DNS name of this "host" for RMR
+
+if [[ -d /playpen/bin ]]		# where python things live since python doesn't use PATH :(
+then
+	PATH=/playpen/bin:$PATH
+	bin_dir=/playpen/bin
+else
+	PATH=$PATH:.
+	bin_dir="."
+fi
+
+# suss out the descriptor file. The env name _should_ be a direct pointer
+# to the file, but it might be just a pointer to the directory :(
+#
+df="${XAPP_DESCRIPTOR_PATH:-/opt/ric/config/config-file.json}"      # default is where helm/kubernetes seems to deposit it
+
+
+if [[ -d $df ]]														# we got a bloody directory... must play hide and go seek
+then
+	echo "[INFO] df ($df) is a directory, sussing out a config file from its depths" >&2
+	if [[ -e $df/config-file.json ]]
+	then
+		echo "[INFO] df found: $df/config-file.json" >&2
+		df=$df/config-file.json
+	else
+		cf=$( ls $df/*.json|head -1 )
+		if [[ -z $cf ]]
+		then
+			echo "[FAIL] no json file found in $df" >&2
+			exit 1
+		fi
+		df=$df/$cf
+		echo "[INFO] df is  a directory, using: $df" >&2
+	fi
+fi
+echo "[INFO] descriptor file: $df" >&2
+
+sf=/tmp/PID$$.cfg			# where shell config dump goes
+touch $sf					# must exist
+if [[ -s $df ]]				# pull config stuff into src file; python can't handle a nil/empty file :(
+then
+	echo "[INFO] xam_register: parsing descriptor: $df" >&2
+
+											# CAUTION: these must be concatinated with a trailing space!
+	config_fields="xapp_name:* "			# create space sep list of fields we need
+	config_fields+="version:* "
+	config_fields+="controls:app_man_url:xam_url "	# xapp manager url for registration if there
+	config_fields+="messaging:ports[]name=rmr-data@port:*"
+
+	python3 $bin_dir/j2src.py debug $df $config_fields >$sf
+else
+	echo "[WARN] descriptor file isn't there or is empty: $df" >&2
+fi
+
+echo "[INFO] sourcing info from config" >&2
+cat $sf
+echo "[INFO] end sourced data" >&2
+. $sf
+# -------------------------------------------------------------------------------
+
+unregister=0				# -U to turn this into an unregister
+forreal=""					# set when no-exec mode enabled
+
+# commandline parms override what we find in the config, so parsed late
+while [[ $1 == -* ]]
+do
+	case $1 in
+		-N)	xapp_name=$2; shift;;
+		-n)	forreal="echo no-execute mode, would run: ";;
+		-u)	xam_url="$2"; shift;;				# mostly for testing, but in a pinch might be useful
+		-U)	unregister=1;;
+		-V) version=$2; shift;;
+
+		*)	echo "[FAIL] unrecognised option: $1" >&2
+			exit 1
+			;;
+	esac
+
+	shift
+done
+
+if [[ $xam_url != "http"* ]]
+then
+	echo "[FAIL] url for xapp manager is not close to being valid: $xam_url" >&2
+	exit 1
+fi
+
+if [[ -z $xapp_name || -z $version ]]
+then
+	echo "[FAIL] could not find xapp name and/or version in the config; not supplied on the command line" >&2
+	exit 1
+fi
+
+if (( unregister ))
+then
+	echo "[INFO] sending unregister to xAPP mgr: $xam_url" >&2
+	app_name=$(    quote appName "$xapp_name" )
+	app_in_name=$( quote appInstanceName  "?????" )
+
+	$forreal curl -X POST "${xam_url:-http://junk-not-supplied}/deregister" -H "accept: application/json" -H "Content-Type: application/json" \
+		-d "{ $app_name, $app_in_name }"
+	rv=$?
+
+	rm -fr /tmp/PID$$.*							# clean things (more important in test env than container)
+	exit $rv
+fi
+
+if [[  -s $df ]]
+then
+	config_junk=$( encode_json.py $df )			# squish the config and escape quotes etc
+else
+	echo "[FAIL] no descriptor file (config) found, or file had no size" >&2
+	exit 1
+fi
+
+
+# these are klunky, but makes quoting junk for the curl command a bit less tedious
+app_name=$(    quote appName "$xapp_name" )
+config_path=$( quote configPath "" )
+app_in_name=$( quote appInstanceName  "?????" )
+http_endpt=$(  quote httpEndpoint "" )
+rmr_endpt=$(   quote rmrEndpoint "$svc_name:$port" )
+config=$(      quote config "$config_junk" )
+
+echo "[INFO] sending register to xAPP mgr: $xam_url" >&2
+$forreal curl -X POST "${xam_url:-http://junk-not-supplied}/register" -H "accept: application/json" -H "Content-Type: application/json" \
+	-d "{ $app_name, $config_path, $app_in_name, $http_endpt, $rmr_endpt, $config }"
+rv=$?	# use curl result as exit value not results of cleanup ops
+
+
+# tidy the space before going
+rm -fr /tmp/PID$$.*
+
+exit $rv
+
+