manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  28         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  29         "github.com/gorilla/mux"
  30         "github.com/prometheus/alertmanager/api/v2/models"
  31         "github.com/stretchr/testify/assert"
  32         "io"
  33         "io/ioutil"
  34         "net"
  35         "net/http"
  36         "net/http/httptest"
  37         "os"
  38         "strconv"
  39         "strings"
  40         "testing"
  41         "time"
  42 )
  43
  44 var alarmManager *AlarmManager
  45 var alarmer *alarm.RICAlarm
  46 var eventChan chan string
  47
  48 // Test cases
  49 func TestMain(M *testing.M) {
  50         alarmManager = NewAlarmManager("localhost:9093", 500)
  51         go alarmManager.Run(false)
  52         time.Sleep(time.Duration(10) * time.Second)
  53
  54         // Wait until RMR is up-and-running
  55         for !xapp.Rmr.IsReady() {
  56                 time.Sleep(time.Duration(1) * time.Second)
  57         }
  58
  59         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  60         alarmManager.alarmClient = alarmer
  61         time.Sleep(time.Duration(5) * time.Second)
  62         eventChan = make(chan string)
  63
  64         os.Exit(M.Run())
  65 }
  66
  67 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  68         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  69         var alarmDefinition alarm.AlarmDefinition
  70         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  71         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
  72         req = mux.SetURLVars(req, vars)
  73         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  74         response := executeRequest(req, handleFunc)
  75         checkResponseCode(t, http.StatusOK, response.Code)
  76         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  77         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  78         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
  79                 t.Errorf("Incorrect alarm definition")
  80         }
  81 }
  82
  83 func TestSetAlarmDefinitions(t *testing.T) {
  84         xapp.Logger.Info("TestSetAlarmDefinitions")
  85         var alarm8004Definition alarm.AlarmDefinition
  86         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  87         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  88         alarm8004Definition.EventType = "Processing error"
  89         alarm8004Definition.OperationInstructions = "Not defined"
  90
  91         var alarm8005Definition alarm.AlarmDefinition
  92         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  93         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  94         alarm8005Definition.EventType = "Communication error"
  95         alarm8005Definition.OperationInstructions = "Not defined"
  96
  97         var alarm8006Definition alarm.AlarmDefinition
  98         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
  99         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 100         alarm8006Definition.EventType = "Communication error"
 101         alarm8006Definition.OperationInstructions = "Not defined"
 102
 103         var alarm8007Definition alarm.AlarmDefinition
 104         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 105         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 106         alarm8007Definition.EventType = "Communication error"
 107         alarm8007Definition.OperationInstructions = "Not defined"
 108
 109         var alarm8008Definition alarm.AlarmDefinition
 110         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 111         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 112         alarm8008Definition.EventType = "storage warning"
 113         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
 114
 115         var alarm8009Definition alarm.AlarmDefinition
 116         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 117         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 118         alarm8009Definition.EventType = "storage warning"
 119         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
 120
 121         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 122         pbodyEn, _ := json.Marshal(pbodyParams)
 123         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 124         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 125         response := executeRequest(req, handleFunc)
 126         status := checkResponseCode(t, http.StatusOK, response.Code)
 127         xapp.Logger.Info("status = %v", status)
 128
 129 }
 130
 131 func TestGetAlarmDefinitions(t *testing.T) {
 132         xapp.Logger.Info("TestGetAlarmDefinitions")
 133         var alarmDefinition alarm.AlarmDefinition
 134         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 135         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 136         req = mux.SetURLVars(req, vars)
 137         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 138         response := executeRequest(req, handleFunc)
 139         checkResponseCode(t, http.StatusOK, response.Code)
 140         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 141         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 142         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 143                 t.Errorf("Incorrect alarm definition")
 144         }
 145 }
 146
 147 func TestDeleteAlarmDefinitions(t *testing.T) {
 148         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 149         //Get all
 150         var ricAlarmDefinitions RicAlarmDefinitions
 151         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 152         req = mux.SetURLVars(req, nil)
 153         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 154         response := executeRequest(req, handleFunc)
 155         checkResponseCode(t, http.StatusOK, response.Code)
 156         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 157         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 158                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 159         }
 160
 161         //Delete 8004
 162         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 163         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 164         req = mux.SetURLVars(req, vars)
 165         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 166         response = executeRequest(req, handleFunc)
 167         checkResponseCode(t, http.StatusOK, response.Code)
 168
 169         //Get 8004 fail
 170         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 171         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 172         req = mux.SetURLVars(req, vars)
 173         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 174         response = executeRequest(req, handleFunc)
 175         checkResponseCode(t, http.StatusBadRequest, response.Code)
 176
 177         //Set 8004 success
 178         var alarm8004Definition alarm.AlarmDefinition
 179         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 180         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 181         alarm8004Definition.EventType = "Processing error"
 182         alarm8004Definition.OperationInstructions = "Not defined"
 183         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 184         pbodyEn, _ := json.Marshal(pbodyParams)
 185         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 186         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 187         response = executeRequest(req, handleFunc)
 188         checkResponseCode(t, http.StatusOK, response.Code)
 189
 190         //Get 8004 success
 191         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 192         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 193         req = mux.SetURLVars(req, vars)
 194         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 195         response = executeRequest(req, handleFunc)
 196         checkResponseCode(t, http.StatusOK, response.Code)
 197 }
 198
 199 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 200         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 201         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 202         defer ts.Close()
 203
 204         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 205         assert.Nil(t, alarmer.Raise(a), "raise failed")
 206
 207         VerifyAlarm(t, a, 1)
 208 }
 209
 210 func TestAlarmClearedSucess(t *testing.T) {
 211         xapp.Logger.Info("TestAlarmClearedSucess")
 212         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 213         defer ts.Close()
 214
 215         // Raise the alarm
 216         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 217         assert.Nil(t, alarmer.Raise(a), "raise failed")
 218
 219         VerifyAlarm(t, a, 1)
 220
 221         // Now Clear the alarm and check alarm is removed
 222         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 223         assert.Nil(t, alarmer.Clear(a), "clear failed")
 224
 225         time.Sleep(time.Duration(2) * time.Second)
 226         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 227 }
 228
 229 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 230         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 231         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 232         defer ts.Close()
 233
 234         // Raise two alarms
 235         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 236         assert.Nil(t, alarmer.Raise(a), "raise failed")
 237
 238         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 239         assert.Nil(t, alarmer.Raise(b), "raise failed")
 240
 241         VerifyAlarm(t, a, 2)
 242         VerifyAlarm(t, b, 2)
 243 }
 244
 245 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 246         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 247         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 248         defer ts.Close()
 249
 250         // Raise two alarms
 251         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 252         assert.Nil(t, alarmer.Clear(a), "clear failed")
 253
 254         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 255         assert.Nil(t, alarmer.Clear(b), "clear failed")
 256
 257         time.Sleep(time.Duration(2) * time.Second)
 258         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 259 }
 260
 261 func TestAlarmsSuppresedSucess(t *testing.T) {
 262         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 263         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 264         defer ts.Close()
 265
 266         // Raise two similar/matching alarms ... the second one suppresed
 267         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 268         assert.Nil(t, alarmer.Raise(a), "raise failed")
 269         assert.Nil(t, alarmer.Raise(a), "raise failed")
 270
 271         VerifyAlarm(t, a, 1)
 272         assert.Nil(t, alarmer.Clear(a), "clear failed")
 273 }
 274
 275 func TestInvalidAlarms(t *testing.T) {
 276         xapp.Logger.Info("TestInvalidAlarms")
 277         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 278         assert.Nil(t, alarmer.Raise(a), "raise failed")
 279         time.Sleep(time.Duration(2) * time.Second)
 280 }
 281
 282 func TestAlarmHandlingErrorCases(t *testing.T) {
 283         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 284         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 285         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 286         assert.Nil(t, ok, "raise failed")
 287 }
 288
 289 func TestConsumeUnknownMessage(t *testing.T) {
 290         xapp.Logger.Info("TestConsumeUnknownMessage")
 291         err := alarmManager.Consume(&xapp.RMRParams{})
 292         assert.Nil(t, err, "raise failed")
 293 }
 294
 295 func TestStatusCallback(t *testing.T) {
 296         xapp.Logger.Info("TestStatusCallback")
 297         assert.Equal(t, true, alarmManager.StatusCB())
 298 }
 299
 300 func TestActiveAlarmMaxThresholds(t *testing.T) {
 301         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 302         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 303         alarmManager.maxActiveAlarms = 0
 304         alarmManager.maxAlarmHistory = 10
 305
 306         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 307         assert.Nil(t, alarmer.Raise(a), "raise failed")
 308
 309         var alarmConfigParams alarm.AlarmConfigParams
 310         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 311         req = mux.SetURLVars(req, nil)
 312         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 313         response := executeRequest(req, handleFunc)
 314
 315         // Check HTTP Status Code
 316         checkResponseCode(t, http.StatusOK, response.Code)
 317
 318         // Decode the json output from handler
 319         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 320         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 321                 t.Errorf("Incorrect alarm thresholds")
 322         }
 323
 324         time.Sleep(time.Duration(1) * time.Second)
 325         alarmManager.maxActiveAlarms = 5000
 326         alarmManager.maxAlarmHistory = 20000
 327         VerifyAlarm(t, a, 2)
 328         VerifyAlarm(t, a, 2)
 329         ts.Close()
 330 }
 331
 332 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 333         receivedAlert := waitForEvent()
 334
 335         assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
 336         _, ok := alarmManager.IsMatchFound(a)
 337         assert.True(t, ok)
 338
 339         return receivedAlert
 340 }
 341
 342 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 343         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 344         //assert.Equal(t, receivedAlert, e)
 345 }
 346
 347 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 348         l, err := net.Listen("tcp", "localhost:9093")
 349         if err != nil {
 350                 t.Error("Failed to create listener: " + err.Error())
 351         }
 352         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 353                 assert.Equal(t, r.Method, method)
 354                 assert.Equal(t, r.URL.String(), url)
 355
 356                 fireEvent(t, r.Body)
 357
 358                 w.Header().Add("Content-Type", "application/json")
 359                 w.WriteHeader(status)
 360                 b, _ := json.Marshal(respData)
 361                 w.Write(b)
 362         }))
 363         ts.Listener.Close()
 364         ts.Listener = l
 365
 366         ts.Start()
 367
 368         return ts
 369 }
 370
 371 func waitForEvent() string {
 372         receivedAlert := <-eventChan
 373         return receivedAlert
 374 }
 375
 376 func fireEvent(t *testing.T, body io.ReadCloser) {
 377         reqBody, err := ioutil.ReadAll(body)
 378         assert.Nil(t, err, "ioutil.ReadAll failed")
 379         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 380
 381         eventChan <- fmt.Sprintf("%s", reqBody)
 382 }
 383
 384 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 385         rr := httptest.NewRecorder()
 386
 387         handleR.ServeHTTP(rr, req)
 388
 389         return rr
 390 }
 391
 392 func checkResponseCode(t *testing.T, expected, actual int) bool {
 393         if expected != actual {
 394                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 395                 return false
 396         }
 397         return true
 398 }