From 33213b53827abe314aa4cdc98c0b9e71e55d53ee Mon Sep 17 00:00:00 2001 From: Mohamed Abukar Date: Wed, 19 Feb 2020 13:00:34 +0200 Subject: [PATCH] [RIC-A-F14] Provide Platform Healthcheck for xApps via O1 Change-Id: I8af87e79998483b104a5386aac1fca8bdb06f178 Signed-off-by: Mohamed Abukar --- Dockerfile | 3 +- agent/cli/o1-cli.go | 40 +++++++++--------- agent/cmd/o1agent.go | 2 +- agent/pkg/nbi/nbi.go | 44 ++++++++++++-------- agent/pkg/nbi/nbi_test.go | 61 +++++++++++++++++++-------- agent/pkg/sbi/sbi.go | 68 +++++++++++++++++++++++++++++++ agent/pkg/sbi/types.go | 2 + agent/yang/o-ran-sc-ric-xapp-desc-v1.yang | 30 ++++++++++++-- container-tag.yaml | 2 +- 9 files changed, 193 insertions(+), 59 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4694ecd..e03eb96 100755 --- a/Dockerfile +++ b/Dockerfile @@ -69,7 +69,8 @@ RUN \ RUN \ cd /opt/dev && \ git clone https://github.com/sysrepo/sysrepo.git && \ - cd sysrepo && mkdir build && cd build && \ + cd sysrepo && sed -i -e 's/2000/30000/g' src/common.h.in && \ + mkdir build && cd build && \ cmake -DCMAKE_BUILD_TYPE:String="Release" -DENABLE_TESTS=OFF -DREPOSITORY_LOC:PATH=/etc/sysrepo .. && \ make -j2 && \ make install && make sr_clean && \ diff --git a/agent/cli/o1-cli.go b/agent/cli/o1-cli.go index 024a6ea..02eca91 100755 --- a/agent/cli/o1-cli.go +++ b/agent/cli/o1-cli.go @@ -1,31 +1,31 @@ package main import ( - "log" - "strings" - "time" - "os" "bytes" + "encoding/json" "flag" "fmt" - "encoding/json" - "io/ioutil" "github.com/Juniper/go-netconf/netconf" xj "github.com/basgys/goxml2json" "golang.org/x/crypto/ssh" + "io/ioutil" + "log" + "os" + "strings" + "time" ) var ( - host = flag.String("host", "localhost", "Hostname") - username = flag.String("username", "netconf", "Username") - passwd = flag.String("password", "netconf", "Password") - source = flag.String("source", "running", "Source datastore") - target = flag.String("target", "running", "Target datastore") - subtree = flag.String("subtree", "netconf-server", "Subtree or module to select") - namespace = flag.String("namespace", "urn:o-ran:ric:gnb-status:1.0", "XML namespace") - file = flag.String("file", "", "Configuration file") - action = flag.String("action", "get", "Netconf command: get or edit") - timeout = flag.Int("timeout", 30, "Timeout") + host = flag.String("host", "localhost", "Hostname") + username = flag.String("username", "netconf", "Username") + passwd = flag.String("password", "netconf", "Password") + source = flag.String("source", "running", "Source datastore") + target = flag.String("target", "running", "Target datastore") + subtree = flag.String("subtree", "ric", "Subtree or module to select") + namespace = flag.String("namespace", "urn:o-ran:ric:xapp-desc:1.0", "XML namespace") + file = flag.String("file", "", "Configuration file") + action = flag.String("action", "get", "Netconf command: get or edit") + timeout = flag.Int("timeout", 30, "Timeout") getStateXml = "" getConfigXml = "<%s/><%s/>" @@ -34,10 +34,10 @@ var ( func main() { defer func() { // catch or finally - if err := recover(); err != nil { //catch - fmt.Fprintf(os.Stderr, "Something went wrong: %v\n", err) - os.Exit(1) - } + if err := recover(); err != nil { //catch + fmt.Fprintf(os.Stderr, "Something went wrong: %v\n", err) + os.Exit(1) + } }() if flag.Parse(); flag.Parsed() == false { diff --git a/agent/cmd/o1agent.go b/agent/cmd/o1agent.go index 17a2685..2423160 100755 --- a/agent/cmd/o1agent.go +++ b/agent/cmd/o1agent.go @@ -20,10 +20,10 @@ package main import ( + "fmt" "os" "os/signal" "syscall" - "fmt" "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/nbi" diff --git a/agent/pkg/nbi/nbi.go b/agent/pkg/nbi/nbi.go index ea5a87b..a7dcccb 100755 --- a/agent/pkg/nbi/nbi.go +++ b/agent/pkg/nbi/nbi.go @@ -151,9 +151,13 @@ func nbiModuleChangeCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.cha changedModule := C.GoString(module) changedXpath := C.GoString(xpath) - log.Info("NBI: Module change callback - event='%d' module=%s xpath=%s reqId=%d", event, changedModule, changedXpath, reqId) + log.Info("NBI: change event='%d' module=%s xpath=%s reqId=%d", event, changedModule, changedXpath, reqId) + if C.SR_EV_CHANGE != event { + log.Info("NBI: Changes finalized!") + return C.SR_ERR_OK + } - if C.SR_EV_CHANGE == event { + if changedModule == "o-ran-sc-ric-xapp-desc-v1" { configJson := C.yang_data_sr2json(session, module, event, &nbiClient.oper) err := nbiClient.ManageXapps(changedModule, C.GoString(configJson), int(nbiClient.oper)) if err != nil { @@ -161,9 +165,9 @@ func nbiModuleChangeCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.cha } } - if C.SR_EV_DONE == event { + if changedModule == "o-ran-sc-ric-ueec-config-v1" { configJson := C.get_data_json(session, module) - err := nbiClient.ManageConfigmaps(changedModule, C.GoString(configJson), int(nbiClient.oper)) + err := nbiClient.ManageConfigmaps(changedModule, C.GoString(configJson), int(C.SR_OP_MODIFIED)) if err != nil { return C.SR_ERR_OPERATION_FAILED } @@ -175,7 +179,7 @@ func nbiModuleChangeCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.cha func (n *Nbi) ManageXapps(module, configJson string, oper int) error { log.Info("ManageXapps: module=%s configJson=%s", module, configJson) - if configJson == "" || module != "o-ran-sc-ric-xapp-desc-v1" { + if configJson == "" { return nil } @@ -207,7 +211,7 @@ func (n *Nbi) ManageXapps(module, configJson string, oper int) error { func (n *Nbi) ManageConfigmaps(module, configJson string, oper int) error { log.Info("ManageConfig: module=%s configJson=%s", module, configJson) - if configJson == "" || module != "o-ran-sc-ric-ueec-config-v1" { + if configJson == "" { return nil } @@ -217,6 +221,7 @@ func (n *Nbi) ManageConfigmaps(module, configJson string, oper int) error { value, err := n.ParseJson(configJson) if err != nil { + log.Info("ParseJson failed with error: %v", oper) return err } @@ -258,7 +263,13 @@ func nbiGnbStateCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.char, r log.Info("NBI: Module state data for module='%s' path='%s' rpath='%s' requested [id=%d]", C.GoString(module), C.GoString(xpath), C.GoString(req_xpath), reqid) if C.GoString(module) == "o-ran-sc-ric-xapp-desc-v1" { - log.Info("xApp health query not implemtented yet!") + podList, _ := sbiClient.GetAllPodStatus("ricxapp") + for _, pod := range podList { + path := fmt.Sprintf("/o-ran-sc-ric-xapp-desc-v1:ric/health/status[name='%s']", pod.Name) + nbiClient.CreateNewElement(session, parent, path, "name", path) + nbiClient.CreateNewElement(session, parent, path, "health", pod.Health) + nbiClient.CreateNewElement(session, parent, path, "status", pod.Status) + } return C.SR_ERR_OK } @@ -282,20 +293,21 @@ func nbiGnbStateCB(session *C.sr_session_ctx_t, module *C.char, xpath *C.char, r log.Info("gNB info: %s -> %s %s %s -> %s %s", ranName, prot, connStat, ntype, gnb.GetGlobalNbId().GetPlmnId(), gnb.GetGlobalNbId().GetNbId()) - nbiClient.CreateNewElement(session, parent, ranName, "ran-name", ranName) - nbiClient.CreateNewElement(session, parent, ranName, "ip", info.Ip) - nbiClient.CreateNewElement(session, parent, ranName, "port", fmt.Sprintf("%d", info.Port)) - nbiClient.CreateNewElement(session, parent, ranName, "plmn-id", gnb.GetGlobalNbId().GetPlmnId()) - nbiClient.CreateNewElement(session, parent, ranName, "nb-id", gnb.GetGlobalNbId().GetNbId()) - nbiClient.CreateNewElement(session, parent, ranName, "e2ap-protocol", prot) - nbiClient.CreateNewElement(session, parent, ranName, "connection-status", connStat) - nbiClient.CreateNewElement(session, parent, ranName, "node", ntype) + path := fmt.Sprintf("/o-ran-sc-ric-gnb-status-v1:ric/nodes/node[ran-name='%s']", ranName) + nbiClient.CreateNewElement(session, parent, path, "ran-name", ranName) + nbiClient.CreateNewElement(session, parent, path, "ip", info.Ip) + nbiClient.CreateNewElement(session, parent, path, "port", fmt.Sprintf("%d", info.Port)) + nbiClient.CreateNewElement(session, parent, path, "plmn-id", gnb.GetGlobalNbId().GetPlmnId()) + nbiClient.CreateNewElement(session, parent, path, "nb-id", gnb.GetGlobalNbId().GetNbId()) + nbiClient.CreateNewElement(session, parent, path, "e2ap-protocol", prot) + nbiClient.CreateNewElement(session, parent, path, "connection-status", connStat) + nbiClient.CreateNewElement(session, parent, path, "node", ntype) } return C.SR_ERR_OK } func (n *Nbi) CreateNewElement(session *C.sr_session_ctx_t, parent **C.char, key, name, value string) { - basePath := fmt.Sprintf("/o-ran-sc-ric-gnb-status-v1:ric/nodes/node[ran-name='%s']/%s", key, name) + basePath := fmt.Sprintf("%s/%s", key, name) log.Info("%s -> %s", basePath, value) cPath := C.CString(basePath) diff --git a/agent/pkg/nbi/nbi_test.go b/agent/pkg/nbi/nbi_test.go index 3812fd6..066a6eb 100755 --- a/agent/pkg/nbi/nbi_test.go +++ b/agent/pkg/nbi/nbi_test.go @@ -20,18 +20,17 @@ package nbi import ( - "os" - "time" "encoding/json" - "testing" - "net" - "net/http" - "net/http/httptest" "github.com/stretchr/testify/assert" + "net" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" apimodel "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/appmgrmodel" "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/sbi" - ) var XappConfig = `{ @@ -67,6 +66,13 @@ var XappDescriptor = `{ } }` +var kpodOutput = ` +NAME READY STATUS RESTARTS AGE +ricxapp-ueec-7bfdd587db-2jl9j 1/1 Running 53 29d +ricxapp-anr-6748846478-8hmtz 1-1 Running 1 29d +ricxapp-dualco-7f76f65c99-5p6c6 0/1 Running 1 29d +` + var n *Nbi // Test cases @@ -122,6 +128,35 @@ func TestGetDeployedXapps(t *testing.T) { assert.Equal(t, true, err == nil) } +func TestGetAllPodStatus(t *testing.T) { + sbi.CommandExec = func(args string) (out string, err error) { + assert.Equal(t, "/usr/local/bin/kubectl get pod -n ricxapp", args) + return kpodOutput, nil + } + + expectedPodList := []sbi.PodStatus{ + sbi.PodStatus{ + Name: "ueec", + Health: "healthy", + Status: "Running", + }, + sbi.PodStatus{ + Name: "anr", + Health: "unavailable", + Status: "Running", + }, + sbi.PodStatus{ + Name: "dualco", + Health: "unhealthy", + Status: "Running", + }, + } + + podList, err := sbiClient.GetAllPodStatus("ricxapp") + assert.Equal(t, true, err == nil) + assert.Equal(t, podList, expectedPodList) +} + func TestErrorCases(t *testing.T) { // Invalid config err := n.ManageXapps("o-ran-sc-ric-xapp-desc-v1", "", 2) @@ -143,14 +178,6 @@ func TestErrorCases(t *testing.T) { err = n.ManageConfigmaps("o-ran-sc-ric-ueec-config-v1", "", 1) assert.Equal(t, true, err == nil) - // Invalid module - err = n.ManageConfigmaps("", "{}", 1) - assert.Equal(t, true, err == nil) - - // Unexpected module - err = n.ManageConfigmaps("o-ran-sc-ric-xapp-desc-v1", "{}", 0) - assert.Equal(t, true, err == nil) - // Invalid operation err = n.ManageConfigmaps("o-ran-sc-ric-ueec-config-v1", "{}", 0) assert.Equal(t, true, err != nil) @@ -188,7 +215,7 @@ func TestTeardown(t *testing.T) { func CreateHTTPServer(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server { l, err := net.Listen("tcp", "localhost:8080") if err != nil { - t.Error("Failed to create listener: " + err.Error()) + t.Error("Failed to create listener: " + err.Error()) } ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, r.Method, method) @@ -216,7 +243,7 @@ func DescMatcher(result, expected *apimodel.XappDescriptor) bool { func ConfigMatcher(result, expected *apimodel.XAppConfig) bool { if *result.Metadata.XappName == *expected.Metadata.XappName && - *result.Metadata.Namespace == *expected.Metadata.Namespace { + *result.Metadata.Namespace == *expected.Metadata.Namespace { return true } return false diff --git a/agent/pkg/sbi/sbi.go b/agent/pkg/sbi/sbi.go index 09dae0e..8801dde 100755 --- a/agent/pkg/sbi/sbi.go +++ b/agent/pkg/sbi/sbi.go @@ -20,8 +20,13 @@ package sbi import ( + "bytes" + "fmt" httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + "os/exec" + "regexp" + "strings" "time" "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" @@ -30,6 +35,12 @@ import ( apimodel "gerrit.oran-osc.org/r/ric-plt/o1mediator/pkg/appmgrmodel" ) +type PodStatus struct { + Name string + Health string + Status string +} + var log = xapp.Logger func NewSBIClient(host, baseUrl string, prot []string, timo int) *SBIClient { @@ -113,3 +124,60 @@ func (s *SBIClient) ModifyXappConfig(xappConfig *apimodel.XAppConfig) error { } return err } + +func (s *SBIClient) GetAllPodStatus(namespace string) ([]PodStatus, error) { + output, err := s.RunCommand(fmt.Sprintf("/usr/local/bin/kubectl get pod -n %s", namespace)) + if err != nil { + return []PodStatus{}, err + } + + podStatusList := []PodStatus{} + var readyStr string + re := regexp.MustCompile(fmt.Sprintf(`%s-.*`, namespace)) + podList := re.FindAllStringSubmatch(string(output), -1) + if podList != nil { + for _, pod := range podList { + p := PodStatus{} + fmt.Sscanf(pod[0], "%s %s %s", &p.Name, &readyStr, &p.Status) + p.Name = strings.Split(p.Name, "-")[1] + p.Health = s.GetHealthState(readyStr) + + podStatusList = append(podStatusList, p) + } + } + return podStatusList, nil +} + +func (s *SBIClient) GetHealthState(ready string) (state string) { + result := strings.Split(ready, "/") + if len(result) < 2 { + return "unavailable" + } + + if result[0] == result[1] { + state = "healthy" + } else { + state = "unhealthy" + } + return +} + +func (s *SBIClient) RunCommand(args string) (string, error) { + return CommandExec(args) +} + +var CommandExec = func(args string) (string, error) { + cmd := exec.Command("/bin/sh", "-c", args) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + xapp.Logger.Debug("Running command: '%s'", cmd) + if err := cmd.Run(); err != nil { + xapp.Logger.Error("Command failed (%s): %v - %s", cmd, err.Error(), stderr.String()) + return "", err + } + xapp.Logger.Debug("Command executed successfully!") + return stdout.String(), nil +} diff --git a/agent/pkg/sbi/types.go b/agent/pkg/sbi/types.go index c15b725..454bbbf 100755 --- a/agent/pkg/sbi/types.go +++ b/agent/pkg/sbi/types.go @@ -40,4 +40,6 @@ type SBIClientInterface interface { BuildXappConfig(name, namespace string, configData interface{}) *apimodel.XAppConfig ModifyXappConfig(xappConfig *apimodel.XAppConfig) error + + GetAllPodStatus(namespace string) ([]PodStatus, error) } diff --git a/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang b/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang index 5c9de38..3a881ee 100755 --- a/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang +++ b/agent/yang/o-ran-sc-ric-xapp-desc-v1.yang @@ -63,16 +63,40 @@ module o-ran-sc-ric-xapp-desc-v1 { "xApp descriptor"; } + typedef health-status { + type enumeration { + enum unavailable { + description + "The health status not available"; + } + enum healthy { + description + "The xApp is healthy"; + } + enum unhealthy { + description + "The xApp is not healthy"; + } + } + description + "xApp health status"; + } + grouping xapp-status { leaf name { type string; description - "Name of the xApp in helm chart"; + "Name of the xApp visible in Kubernetes"; } leaf status { - type boolean; + type string; + description + "The status of the xApp pod: running, restarted, etc"; + } + leaf health { + type health-status; description - "The status of xApp: true=healthy false=not-healthy"; + "The health status of xApp: healthy, not-healthy, unavailable"; } description "xApp health status"; diff --git a/container-tag.yaml b/container-tag.yaml index db88a6c..dc6f324 100644 --- a/container-tag.yaml +++ b/container-tag.yaml @@ -2,4 +2,4 @@ # By default this file is in the docker build directory, # but the location can configured in the JJB template. --- -tag: 0.3.2 +tag: 0.4.0 -- 2.16.6