manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  28         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  29         "github.com/gorilla/mux"
  30         "github.com/prometheus/alertmanager/api/v2/models"
  31         "github.com/stretchr/testify/assert"
  32         "io"
  33         "io/ioutil"
  34         "net"
  35         "net/http"
  36         "net/http/httptest"
  37         "os"
  38         "strconv"
  39         "strings"
  40         "testing"
  41         "time"
  42 )
  43
  44 var alarmManager *AlarmManager
  45 var alarmer *alarm.RICAlarm
  46 var eventChan chan string
  47
  48 // Test cases
  49 func TestMain(M *testing.M) {
  50         alarmManager = NewAlarmManager("localhost:9093", 500)
  51         alarmManager.alertInterval = 20000
  52         go alarmManager.Run(false)
  53         time.Sleep(time.Duration(10) * time.Second)
  54
  55         // Wait until RMR is up-and-running
  56         for !xapp.Rmr.IsReady() {
  57                 time.Sleep(time.Duration(1) * time.Second)
  58         }
  59
  60         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  61         alarmManager.alarmClient = alarmer
  62         time.Sleep(time.Duration(5) * time.Second)
  63         eventChan = make(chan string)
  64
  65         os.Exit(M.Run())
  66 }
  67
  68 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  69         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  70         var alarmDefinition alarm.AlarmDefinition
  71         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  72         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
  73         req = mux.SetURLVars(req, vars)
  74         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  75         response := executeRequest(req, handleFunc)
  76         checkResponseCode(t, http.StatusOK, response.Code)
  77         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  78         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  79         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
  80                 t.Errorf("Incorrect alarm definition")
  81         }
  82 }
  83
  84 func TestSetAlarmDefinitions(t *testing.T) {
  85         xapp.Logger.Info("TestSetAlarmDefinitions")
  86         var alarm8004Definition alarm.AlarmDefinition
  87         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  88         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  89         alarm8004Definition.EventType = "Processing error"
  90         alarm8004Definition.OperationInstructions = "Not defined"
  91
  92         var alarm8005Definition alarm.AlarmDefinition
  93         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  94         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  95         alarm8005Definition.EventType = "Communication error"
  96         alarm8005Definition.OperationInstructions = "Not defined"
  97
  98         var alarm8006Definition alarm.AlarmDefinition
  99         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
 100         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 101         alarm8006Definition.EventType = "Communication error"
 102         alarm8006Definition.OperationInstructions = "Not defined"
 103
 104         var alarm8007Definition alarm.AlarmDefinition
 105         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 106         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 107         alarm8007Definition.EventType = "Communication error"
 108         alarm8007Definition.OperationInstructions = "Not defined"
 109
 110         var alarm8008Definition alarm.AlarmDefinition
 111         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 112         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 113         alarm8008Definition.EventType = "storage warning"
 114         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
 115
 116         var alarm8009Definition alarm.AlarmDefinition
 117         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 118         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 119         alarm8009Definition.EventType = "storage warning"
 120         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
 121
 122         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 123         pbodyEn, _ := json.Marshal(pbodyParams)
 124         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 125         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 126         response := executeRequest(req, handleFunc)
 127         status := checkResponseCode(t, http.StatusOK, response.Code)
 128         xapp.Logger.Info("status = %v", status)
 129
 130 }
 131
 132 func TestGetAlarmDefinitions(t *testing.T) {
 133         xapp.Logger.Info("TestGetAlarmDefinitions")
 134         var alarmDefinition alarm.AlarmDefinition
 135         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 136         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 137         req = mux.SetURLVars(req, vars)
 138         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 139         response := executeRequest(req, handleFunc)
 140         checkResponseCode(t, http.StatusOK, response.Code)
 141         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 142         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 143         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 144                 t.Errorf("Incorrect alarm definition")
 145         }
 146 }
 147
 148 func TestDeleteAlarmDefinitions(t *testing.T) {
 149         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 150         //Get all
 151         var ricAlarmDefinitions RicAlarmDefinitions
 152         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 153         req = mux.SetURLVars(req, nil)
 154         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 155         response := executeRequest(req, handleFunc)
 156         checkResponseCode(t, http.StatusOK, response.Code)
 157         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 158         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 159                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 160         }
 161
 162         //Delete 8004
 163         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 164         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 165         req = mux.SetURLVars(req, vars)
 166         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 167         response = executeRequest(req, handleFunc)
 168         checkResponseCode(t, http.StatusOK, response.Code)
 169
 170         //Get 8004 fail
 171         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 172         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 173         req = mux.SetURLVars(req, vars)
 174         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 175         response = executeRequest(req, handleFunc)
 176         checkResponseCode(t, http.StatusBadRequest, response.Code)
 177
 178         //Set 8004 success
 179         var alarm8004Definition alarm.AlarmDefinition
 180         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 181         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 182         alarm8004Definition.EventType = "Processing error"
 183         alarm8004Definition.OperationInstructions = "Not defined"
 184         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 185         pbodyEn, _ := json.Marshal(pbodyParams)
 186         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 187         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 188         response = executeRequest(req, handleFunc)
 189         checkResponseCode(t, http.StatusOK, response.Code)
 190
 191         //Get 8004 success
 192         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 193         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 194         req = mux.SetURLVars(req, vars)
 195         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 196         response = executeRequest(req, handleFunc)
 197         checkResponseCode(t, http.StatusOK, response.Code)
 198 }
 199
 200 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 201         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 202         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 203         defer ts.Close()
 204
 205         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 206         assert.Nil(t, alarmer.Raise(a), "raise failed")
 207
 208         VerifyAlarm(t, a, 1)
 209 }
 210
 211 func TestAlarmClearedSucess(t *testing.T) {
 212         xapp.Logger.Info("TestAlarmClearedSucess")
 213         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 214         defer ts.Close()
 215
 216         // Raise the alarm
 217         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 218         assert.Nil(t, alarmer.Raise(a), "raise failed")
 219
 220         VerifyAlarm(t, a, 1)
 221
 222         // Now Clear the alarm and check alarm is removed
 223         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 224         assert.Nil(t, alarmer.Clear(a), "clear failed")
 225
 226         time.Sleep(time.Duration(2) * time.Second)
 227         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 228 }
 229
 230 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 231         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 232         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 233         defer ts.Close()
 234
 235         // Raise two alarms
 236         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 237         assert.Nil(t, alarmer.Raise(a), "raise failed")
 238
 239         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 240         assert.Nil(t, alarmer.Raise(b), "raise failed")
 241
 242         time.Sleep(time.Duration(2) * time.Second)
 243         VerifyAlarm(t, a, 2)
 244         VerifyAlarm(t, b, 2)
 245 }
 246
 247 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 248         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 249         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 250         defer ts.Close()
 251
 252         // Raise two alarms
 253         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 254         assert.Nil(t, alarmer.Clear(a), "clear failed")
 255
 256         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 257         assert.Nil(t, alarmer.Clear(b), "clear failed")
 258
 259         time.Sleep(time.Duration(2) * time.Second)
 260         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 261 }
 262
 263 func TestAlarmsSuppresedSucess(t *testing.T) {
 264         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 265         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 266         defer ts.Close()
 267
 268         // Raise two similar/matching alarms ... the second one suppresed
 269         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 270         assert.Nil(t, alarmer.Raise(a), "raise failed")
 271         assert.Nil(t, alarmer.Raise(a), "raise failed")
 272
 273         VerifyAlarm(t, a, 1)
 274         assert.Nil(t, alarmer.Clear(a), "clear failed")
 275 }
 276
 277 func TestInvalidAlarms(t *testing.T) {
 278         xapp.Logger.Info("TestInvalidAlarms")
 279         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 280         assert.Nil(t, alarmer.Raise(a), "raise failed")
 281         time.Sleep(time.Duration(2) * time.Second)
 282 }
 283
 284 func TestAlarmHandlingErrorCases(t *testing.T) {
 285         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 286         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 287         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 288         assert.Nil(t, ok, "raise failed")
 289 }
 290
 291 func TestConsumeUnknownMessage(t *testing.T) {
 292         xapp.Logger.Info("TestConsumeUnknownMessage")
 293         err := alarmManager.Consume(&xapp.RMRParams{})
 294         assert.Nil(t, err, "raise failed")
 295 }
 296
 297 func TestStatusCallback(t *testing.T) {
 298         xapp.Logger.Info("TestStatusCallback")
 299         assert.Equal(t, true, alarmManager.StatusCB())
 300 }
 301
 302 func TestActiveAlarmMaxThresholds(t *testing.T) {
 303         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 304         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 305         alarmManager.maxActiveAlarms = 0
 306         alarmManager.maxAlarmHistory = 10
 307
 308         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 309         assert.Nil(t, alarmer.Raise(a), "raise failed")
 310
 311         var alarmConfigParams alarm.AlarmConfigParams
 312         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 313         req = mux.SetURLVars(req, nil)
 314         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 315         response := executeRequest(req, handleFunc)
 316
 317         // Check HTTP Status Code
 318         checkResponseCode(t, http.StatusOK, response.Code)
 319
 320         // Decode the json output from handler
 321         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 322         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 323                 t.Errorf("Incorrect alarm thresholds")
 324         }
 325
 326         time.Sleep(time.Duration(1) * time.Second)
 327         alarmManager.maxActiveAlarms = 5000
 328         alarmManager.maxAlarmHistory = 20000
 329         VerifyAlarm(t, a, 2)
 330         VerifyAlarm(t, a, 2)
 331         ts.Close()
 332 }
 333
 334 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 335         receivedAlert := waitForEvent()
 336
 337         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 338         _, ok := alarmManager.IsMatchFound(a)
 339         assert.True(t, ok)
 340
 341         return receivedAlert
 342 }
 343
 344 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 345         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 346         //assert.Equal(t, receivedAlert, e)
 347 }
 348
 349 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 350         l, err := net.Listen("tcp", "localhost:9093")
 351         if err != nil {
 352                 t.Error("Failed to create listener: " + err.Error())
 353         }
 354         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 355                 assert.Equal(t, r.Method, method)
 356                 assert.Equal(t, r.URL.String(), url)
 357
 358                 fireEvent(t, r.Body)
 359
 360                 w.Header().Add("Content-Type", "application/json")
 361                 w.WriteHeader(status)
 362                 b, _ := json.Marshal(respData)
 363                 w.Write(b)
 364         }))
 365         ts.Listener.Close()
 366         ts.Listener = l
 367
 368         ts.Start()
 369
 370         return ts
 371 }
 372
 373 func waitForEvent() string {
 374         receivedAlert := <-eventChan
 375         return receivedAlert
 376 }
 377
 378 func fireEvent(t *testing.T, body io.ReadCloser) {
 379         reqBody, err := ioutil.ReadAll(body)
 380         assert.Nil(t, err, "ioutil.ReadAll failed")
 381         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 382
 383         eventChan <- fmt.Sprintf("%s", reqBody)
 384 }
 385
 386 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 387         rr := httptest.NewRecorder()
 388
 389         handleR.ServeHTTP(rr, req)
 390
 391         return rr
 392 }
 393
 394 func checkResponseCode(t *testing.T, expected, actual int) bool {
 395         if expected != actual {
 396                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 397                 return false
 398         }
 399         return true
 400 }