[RIC-A-F14] Provide Platform Healthcheck for xApps via O1

Change-Id: I8af87e79998483b104a5386aac1fca8bdb06f178
Signed-off-by: Mohamed Abukar <abukar.mohamed@nokia.com>

diff --git a/Dockerfile b/Dockerfile
index 4694ecd..e03eb96 100755 (executable)
--- a/Dockerfile
+++ b/Dockerfile
@@ -69,7 +69,8 @@ RUN \
 RUN \
       cd /opt/dev && \
       git clone https://github.com/sysrepo/sysrepo.git && \
-      cd sysrepo && mkdir build && cd build && \
+      cd sysrepo && sed -i -e 's/2000/30000/g' src/common.h.in && \
+      mkdir build && cd build && \
       cmake -DCMAKE_BUILD_TYPE:String="Release" -DENABLE_TESTS=OFF -DREPOSITORY_LOC:PATH=/etc/sysrepo .. && \
       make -j2 && \
       make install && make sr_clean && \

diff --git a/agent/cli/o1-cli.go b/agent/cli/o1-cli.go
index 024a6ea..02eca91 100755 (executable)
--- a/agent/cli/o1-cli.go
+++ b/agent/cli/o1-cli.go
@@ -1,31 +1,31 @@
 package main
 
 import (
-       "log"
-       "strings"
-       "time"
-       "os"
        "bytes"
+       "encoding/json"
        "flag"
        "fmt"
-       "encoding/json"
-       "io/ioutil"
        "github.com/Juniper/go-netconf/netconf"
        xj "github.com/basgys/goxml2json"
        "golang.org/x/crypto/ssh"
+       "io/ioutil"
+       "log"
+       "os"
+       "strings"
+       "time"
 )
 
 var (
-       host     = flag.String("host", "localhost", "Hostname")
-       username = flag.String("username", "netconf", "Username")
-       passwd   = flag.String("password", "netconf", "Password")
-       source   = flag.String("source", "running", "Source datastore")
-       target   = flag.String("target", "running", "Target datastore")
-       subtree  = flag.String("subtree", "netconf-server", "Subtree or module to select")
-       namespace = flag.String("namespace", "urn:o-ran:ric:gnb-status:1.0", "XML namespace")
-       file     = flag.String("file", "", "Configuration file")
-       action   = flag.String("action", "get", "Netconf command: get or edit")
-       timeout  = flag.Int("timeout", 30, "Timeout")
+       host      = flag.String("host", "localhost", "Hostname")
+       username  = flag.String("username", "netconf", "Username")
+       passwd    = flag.String("password", "netconf", "Password")
+       source    = flag.String("source", "running", "Source datastore")
+       target    = flag.String("target", "running", "Target datastore")
+       subtree   = flag.String("subtree", "ric", "Subtree or module to select")
+       namespace = flag.String("namespace", "urn:o-ran:ric:xapp-desc:1.0", "XML namespace")
+       file      = flag.String("file", "", "Configuration file")
+       action    = flag.String("action", "get", "Netconf command: get or edit")
+       timeout   = flag.Int("timeout", 30, "Timeout")
 
        getStateXml   = "<get><filter type=\"subtree\"><ric xmlns=\"%s\"></ric></filter></get>"
        getConfigXml  = "<get-config><source><%s/></source><filter type=\"subtree\"><%s/></filter></get-config>"
@@ -34,10 +34,10 @@ var (
 
 func main() {
        defer func() { // catch or finally
-        if err := recover(); err != nil { //catch
-            fmt.Fprintf(os.Stderr, "Something went wrong: %v\n", err)
-            os.Exit(1)
-        }
+               if err := recover(); err != nil { //catch
+                       fmt.Fprintf(os.Stderr, "Something went wrong: %v\n", err)
+                       os.Exit(1)
+               }
        }()
 
        if flag.Parse(); flag.Parsed() == false {

diff --git a/agent/cmd/o1agent.go b/agent/cmd/o1agent.go
index 17a2685..2423160 100755 (executable)
--- a/agent/cmd/o1agent.go
+++ b/agent/cmd/o1agent.go
@@ -20,10 +20,10 @@
 package main
 
 import (
+       "fmt"
        "os"
        "os/signal"
        "syscall"
-       "fmt"
 
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
        "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/nbi"

diff --git a/agent/pkg/nbi/nbi.go b/agent/pkg/nbi/nbi.go
index ea5a87b..a7dcccb 100755 (executable)
--- a/agent/pkg/nbi/nbi.go
+++ b/agent/pkg/nbi/nbi.go
@@ -151,9 +151,13 @@ func nbiModuleChangeCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.cha
        changedModule := C.GoString(module)
        changedXpath := C.GoString(xpath)
 
-       log.Info("NBI: Module change callback - event='%d' module=%s xpath=%s reqId=%d", event, changedModule, changedXpath, reqId)
+       log.Info("NBI: change event='%d' module=%s xpath=%s reqId=%d", event, changedModule, changedXpath, reqId)
+       if C.SR_EV_CHANGE != event {
+               log.Info("NBI: Changes finalized!")
+               return C.SR_ERR_OK
+       }
 
-       if C.SR_EV_CHANGE == event {
+       if changedModule == "o-ran-sc-ric-xapp-desc-v1" {
                configJson := C.yang_data_sr2json(session, module, event, &nbiClient.oper)
                err := nbiClient.ManageXapps(changedModule, C.GoString(configJson), int(nbiClient.oper))
                if err != nil {
@@ -161,9 +165,9 @@ func nbiModuleChangeCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.cha
                }
        }
 
-       if C.SR_EV_DONE == event {
+       if changedModule == "o-ran-sc-ric-ueec-config-v1" {
                configJson := C.get_data_json(session, module)
-               err := nbiClient.ManageConfigmaps(changedModule, C.GoString(configJson), int(nbiClient.oper))
+               err := nbiClient.ManageConfigmaps(changedModule, C.GoString(configJson), int(C.SR_OP_MODIFIED))
                if err != nil {
                        return C.SR_ERR_OPERATION_FAILED
                }
@@ -175,7 +179,7 @@ func nbiModuleChangeCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.cha
 func (n *Nbi) ManageXapps(module, configJson string, oper int) error {
        log.Info("ManageXapps: module=%s configJson=%s", module, configJson)
 
-       if configJson == "" || module != "o-ran-sc-ric-xapp-desc-v1" {
+       if configJson == "" {
                return nil
        }
 
@@ -207,7 +211,7 @@ func (n *Nbi) ManageXapps(module, configJson string, oper int) error {
 func (n *Nbi) ManageConfigmaps(module, configJson string, oper int) error {
        log.Info("ManageConfig: module=%s configJson=%s", module, configJson)
 
-       if configJson == "" || module != "o-ran-sc-ric-ueec-config-v1" {
+       if configJson == "" {
                return nil
        }
 
@@ -217,6 +221,7 @@ func (n *Nbi) ManageConfigmaps(module, configJson string, oper int) error {
 
        value, err := n.ParseJson(configJson)
        if err != nil {
+               log.Info("ParseJson failed with error: %v", oper)
                return err
        }
 
@@ -258,7 +263,13 @@ func nbiGnbStateCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.char, r
        log.Info("NBI: Module state data for module='%s' path='%s' rpath='%s' requested [id=%d]", C.GoString(module), C.GoString(xpath), C.GoString(req_xpath), reqid)
 
        if C.GoString(module) == "o-ran-sc-ric-xapp-desc-v1" {
-               log.Info("xApp health query not implemtented yet!")
+               podList, _ := sbiClient.GetAllPodStatus("ricxapp")
+               for _, pod := range podList {
+                       path := fmt.Sprintf("/o-ran-sc-ric-xapp-desc-v1:ric/health/status[name='%s']", pod.Name)
+                       nbiClient.CreateNewElement(session, parent, path, "name", path)
+                       nbiClient.CreateNewElement(session, parent, path, "health", pod.Health)
+                       nbiClient.CreateNewElement(session, parent, path, "status", pod.Status)
+               }
                return C.SR_ERR_OK
        }
 
@@ -282,20 +293,21 @@ func nbiGnbStateCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.char, r
 
                log.Info("gNB info: %s -> %s %s %s -> %s %s", ranName, prot, connStat, ntype, gnb.GetGlobalNbId().GetPlmnId(), gnb.GetGlobalNbId().GetNbId())
 
-               nbiClient.CreateNewElement(session, parent, ranName, "ran-name", ranName)
-               nbiClient.CreateNewElement(session, parent, ranName, "ip", info.Ip)
-               nbiClient.CreateNewElement(session, parent, ranName, "port", fmt.Sprintf("%d", info.Port))
-               nbiClient.CreateNewElement(session, parent, ranName, "plmn-id", gnb.GetGlobalNbId().GetPlmnId())
-               nbiClient.CreateNewElement(session, parent, ranName, "nb-id", gnb.GetGlobalNbId().GetNbId())
-               nbiClient.CreateNewElement(session, parent, ranName, "e2ap-protocol", prot)
-               nbiClient.CreateNewElement(session, parent, ranName, "connection-status", connStat)
-               nbiClient.CreateNewElement(session, parent, ranName, "node", ntype)
+               path := fmt.Sprintf("/o-ran-sc-ric-gnb-status-v1:ric/nodes/node[ran-name='%s']", ranName)
+               nbiClient.CreateNewElement(session, parent, path, "ran-name", ranName)
+               nbiClient.CreateNewElement(session, parent, path, "ip", info.Ip)
+               nbiClient.CreateNewElement(session, parent, path, "port", fmt.Sprintf("%d", info.Port))
+               nbiClient.CreateNewElement(session, parent, path, "plmn-id", gnb.GetGlobalNbId().GetPlmnId())
+               nbiClient.CreateNewElement(session, parent, path, "nb-id", gnb.GetGlobalNbId().GetNbId())
+               nbiClient.CreateNewElement(session, parent, path, "e2ap-protocol", prot)
+               nbiClient.CreateNewElement(session, parent, path, "connection-status", connStat)
+               nbiClient.CreateNewElement(session, parent, path, "node", ntype)
        }
        return C.SR_ERR_OK
 }
 
 func (n *Nbi) CreateNewElement(session *C.sr_session_ctx_t, parent **C.char, key, name, value string) {
-       basePath := fmt.Sprintf("/o-ran-sc-ric-gnb-status-v1:ric/nodes/node[ran-name='%s']/%s", key, name)
+       basePath := fmt.Sprintf("%s/%s", key, name)
        log.Info("%s -> %s", basePath, value)
 
        cPath := C.CString(basePath)

diff --git a/agent/pkg/nbi/nbi_test.go b/agent/pkg/nbi/nbi_test.go
index 3812fd6..066a6eb 100755 (executable)
--- a/agent/pkg/nbi/nbi_test.go
+++ b/agent/pkg/nbi/nbi_test.go
@@ -20,18 +20,17 @@
 package nbi
 
 import (
-       "os"    
-       "time"
        "encoding/json"
-       "testing"
-       "net"
-    "net/http"
-    "net/http/httptest"
        "github.com/stretchr/testify/assert"
+       "net"
+       "net/http"
+       "net/http/httptest"
+       "os"
+       "testing"
+       "time"
 
        apimodel "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/appmgrmodel"
        "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/sbi"
-       
 )
 
 var XappConfig = `{
@@ -67,6 +66,13 @@ var XappDescriptor = `{
        }
   }`
 
+var kpodOutput = `
+NAME                               READY   STATUS    RESTARTS   AGE
+ricxapp-ueec-7bfdd587db-2jl9j      1/1     Running   53         29d
+ricxapp-anr-6748846478-8hmtz       1-1     Running   1          29d
+ricxapp-dualco-7f76f65c99-5p6c6    0/1     Running   1          29d
+`
+
 var n *Nbi
 
 // Test cases
@@ -122,6 +128,35 @@ func TestGetDeployedXapps(t *testing.T) {
        assert.Equal(t, true, err == nil)
 }
 
+func TestGetAllPodStatus(t *testing.T) {
+       sbi.CommandExec = func(args string) (out string, err error) {
+               assert.Equal(t, "/usr/local/bin/kubectl get pod -n ricxapp", args)
+               return kpodOutput, nil
+       }
+
+       expectedPodList := []sbi.PodStatus{
+               sbi.PodStatus{
+                       Name:   "ueec",
+                       Health: "healthy",
+                       Status: "Running",
+               },
+               sbi.PodStatus{
+                       Name:   "anr",
+                       Health: "unavailable",
+                       Status: "Running",
+               },
+               sbi.PodStatus{
+                       Name:   "dualco",
+                       Health: "unhealthy",
+                       Status: "Running",
+               },
+       }
+
+       podList, err := sbiClient.GetAllPodStatus("ricxapp")
+       assert.Equal(t, true, err == nil)
+       assert.Equal(t, podList, expectedPodList)
+}
+
 func TestErrorCases(t *testing.T) {
        // Invalid config
        err := n.ManageXapps("o-ran-sc-ric-xapp-desc-v1", "", 2)
@@ -143,14 +178,6 @@ func TestErrorCases(t *testing.T) {
        err = n.ManageConfigmaps("o-ran-sc-ric-ueec-config-v1", "", 1)
        assert.Equal(t, true, err == nil)
 
-       // Invalid module
-       err = n.ManageConfigmaps("", "{}", 1)
-       assert.Equal(t, true, err == nil)
-
-       // Unexpected module
-       err = n.ManageConfigmaps("o-ran-sc-ric-xapp-desc-v1", "{}", 0)
-       assert.Equal(t, true, err == nil)
-
        // Invalid operation
        err = n.ManageConfigmaps("o-ran-sc-ric-ueec-config-v1", "{}", 0)
        assert.Equal(t, true, err != nil)
@@ -188,7 +215,7 @@ func TestTeardown(t *testing.T) {
 func CreateHTTPServer(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
        l, err := net.Listen("tcp", "localhost:8080")
        if err != nil {
-                       t.Error("Failed to create listener: " + err.Error())
+               t.Error("Failed to create listener: " + err.Error())
        }
        ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
                assert.Equal(t, r.Method, method)
@@ -216,7 +243,7 @@ func DescMatcher(result, expected *apimodel.XappDescriptor) bool {
 
 func ConfigMatcher(result, expected *apimodel.XAppConfig) bool {
        if *result.Metadata.XappName == *expected.Metadata.XappName &&
-          *result.Metadata.Namespace == *expected.Metadata.Namespace {
+               *result.Metadata.Namespace == *expected.Metadata.Namespace {
                return true
        }
        return false

diff --git a/agent/pkg/sbi/sbi.go b/agent/pkg/sbi/sbi.go
index 09dae0e..8801dde 100755 (executable)
--- a/agent/pkg/sbi/sbi.go
+++ b/agent/pkg/sbi/sbi.go
@@ -20,8 +20,13 @@
 package sbi
 
 import (
+       "bytes"
+       "fmt"
        httptransport "github.com/go-openapi/runtime/client"
        "github.com/go-openapi/strfmt"
+       "os/exec"
+       "regexp"
+       "strings"
        "time"
 
        "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
@@ -30,6 +35,12 @@ import (
        apimodel "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/appmgrmodel"
 )
 
+type PodStatus struct {
+       Name   string
+       Health string
+       Status string
+}
+
 var log = xapp.Logger
 
 func NewSBIClient(host, baseUrl string, prot []string, timo int) *SBIClient {
@@ -113,3 +124,60 @@ func (s *SBIClient) ModifyXappConfig(xappConfig *apimodel.XAppConfig) error {
        }
        return err
 }
+
+func (s *SBIClient) GetAllPodStatus(namespace string) ([]PodStatus, error) {
+       output, err := s.RunCommand(fmt.Sprintf("/usr/local/bin/kubectl get pod -n %s", namespace))
+       if err != nil {
+               return []PodStatus{}, err
+       }
+
+       podStatusList := []PodStatus{}
+       var readyStr string
+       re := regexp.MustCompile(fmt.Sprintf(`%s-.*`, namespace))
+       podList := re.FindAllStringSubmatch(string(output), -1)
+       if podList != nil {
+               for _, pod := range podList {
+                       p := PodStatus{}
+                       fmt.Sscanf(pod[0], "%s %s %s", &p.Name, &readyStr, &p.Status)
+                       p.Name = strings.Split(p.Name, "-")[1]
+                       p.Health = s.GetHealthState(readyStr)
+
+                       podStatusList = append(podStatusList, p)
+               }
+       }
+       return podStatusList, nil
+}
+
+func (s *SBIClient) GetHealthState(ready string) (state string) {
+       result := strings.Split(ready, "/")
+       if len(result) < 2 {
+               return "unavailable"
+       }
+
+       if result[0] == result[1] {
+               state = "healthy"
+       } else {
+               state = "unhealthy"
+       }
+       return
+}
+
+func (s *SBIClient) RunCommand(args string) (string, error) {
+       return CommandExec(args)
+}
+
+var CommandExec = func(args string) (string, error) {
+       cmd := exec.Command("/bin/sh", "-c", args)
+       var stdout bytes.Buffer
+       var stderr bytes.Buffer
+       cmd.Stdout = &stdout
+       cmd.Stderr = &stderr
+
+       xapp.Logger.Debug("Running command: '%s'", cmd)
+       if err := cmd.Run(); err != nil {
+               xapp.Logger.Error("Command failed (%s): %v - %s", cmd, err.Error(), stderr.String())
+               return "", err
+       }
+       xapp.Logger.Debug("Command executed successfully!")
+       return stdout.String(), nil
+}

diff --git a/agent/pkg/sbi/types.go b/agent/pkg/sbi/types.go
index c15b725..454bbbf 100755 (executable)
--- a/agent/pkg/sbi/types.go
+++ b/agent/pkg/sbi/types.go
@@ -40,4 +40,6 @@ type SBIClientInterface interface {
 
        BuildXappConfig(name, namespace string, configData interface{}) *apimodel.XAppConfig
        ModifyXappConfig(xappConfig *apimodel.XAppConfig) error
+
+       GetAllPodStatus(namespace string) ([]PodStatus, error)
 }

diff --git a/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang b/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang
index 5c9de38..3a881ee 100755 (executable)
--- a/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang
+++ b/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang
@@ -63,16 +63,40 @@ module o-ran-sc-ric-xapp-desc-v1 {
             "xApp descriptor";
     }
 
+    typedef health-status {
+         type enumeration {
+            enum unavailable {
+                description
+                    "The health status not available";
+            }
+            enum healthy {
+                description
+                    "The xApp is healthy";
+            }
+            enum unhealthy {
+                description
+                    "The xApp is not healthy";
+            }
+        }
+        description
+            "xApp health status";
+    }
+
     grouping xapp-status {
         leaf name {
             type string;
             description
-                "Name of the xApp in helm chart";
+                "Name of the xApp visible in Kubernetes";
         }
         leaf status {
-            type boolean;
+            type string;
+            description
+                "The status of the xApp pod: running, restarted, etc";
+        }
+        leaf health {
+            type health-status;
             description
-                "The status of xApp: true=healthy false=not-healthy";
+                "The health status of xApp: healthy, not-healthy, unavailable";
         }
         description
             "xApp health status";

diff --git a/container-tag.yaml b/container-tag.yaml
index db88a6c..dc6f324 100644 (file)
--- a/container-tag.yaml
+++ b/container-tag.yaml
@@ -2,4 +2,4 @@
 # By default this file is in the docker build directory,
 # but the location can configured in the JJB template.
 ---
-tag: 0.3.2
+tag: 0.4.0