manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "encoding/json"
  25         "fmt"
  26         "github.com/stretchr/testify/assert"
  27         "io"
  28         "io/ioutil"
  29         "net"
  30         "net/http"
  31         "net/http/httptest"
  32         "os"
  33         "strings"
  34         "testing"
  35         "time"
  36         "github.com/gorilla/mux"
  37         "strconv"
  38         "bytes"
  39         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  40         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  41         "github.com/prometheus/alertmanager/api/v2/models"
  42 )
  43
  44 var alarmManager *AlarmManager
  45 var alarmer *alarm.RICAlarm
  46 var eventChan chan string
  47
  48 // Test cases
  49 func TestMain(M *testing.M) {
  50         os.Setenv("ALARM_IF_RMR", "true")
  51         alarmManager = NewAlarmManager("localhost:9093", 500)
  52         go alarmManager.Run(false)
  53         time.Sleep(time.Duration(2) * time.Second)
  54
  55         // Wait until RMR is up-and-running
  56         for !xapp.Rmr.IsReady() {
  57                 time.Sleep(time.Duration(1) * time.Second)
  58         }
  59
  60         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  61         alarmManager.alarmClient = alarmer
  62         time.Sleep(time.Duration(5) * time.Second)
  63         eventChan = make(chan string)
  64
  65         os.Exit(M.Run())
  66 }
  67
  68 func TestSetAlarmDefinitions(t *testing.T) {
  69         xapp.Logger.Info("TestSetAlarmDefinitions")
  70         var alarm8004Definition alarm.AlarmDefinition
  71         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  72         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  73         alarm8004Definition.EventType = "Processing error"
  74         alarm8004Definition.OperationInstructions = "Not defined"
  75
  76         var alarm8005Definition alarm.AlarmDefinition
  77         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  78         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  79         alarm8005Definition.EventType = "Communication error"
  80         alarm8005Definition.OperationInstructions = "Not defined"
  81
  82         var alarm8006Definition alarm.AlarmDefinition
  83         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
  84         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
  85         alarm8006Definition.EventType = "Communication error"
  86         alarm8006Definition.OperationInstructions = "Not defined"
  87
  88         var alarm8007Definition alarm.AlarmDefinition
  89         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
  90         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
  91         alarm8007Definition.EventType = "Communication error"
  92         alarm8007Definition.OperationInstructions = "Not defined"
  93
  94         var alarm8008Definition alarm.AlarmDefinition
  95         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
  96         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
  97         alarm8008Definition.EventType = "storage warning"
  98         alarm8008Definition.OperationInstructions = "clear alarms or raise threshold"
  99
 100         var alarm8009Definition alarm.AlarmDefinition
 101         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 102         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 103         alarm8009Definition.EventType = "storage warning"
 104         alarm8009Definition.OperationInstructions = "clear alarms or raise threshold"
 105
 106         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 107         pbodyEn, _ := json.Marshal(pbodyParams)
 108         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 109         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 110         response := executeRequest(req, handleFunc)
 111         status := checkResponseCode(t, http.StatusOK, response.Code)
 112         xapp.Logger.Info("status = %v", status)
 113
 114 }
 115
 116 func TestGetAlarmDefinitions(t *testing.T) {
 117         xapp.Logger.Info("TestGetAlarmDefinitions")
 118         var alarmDefinition alarm.AlarmDefinition
 119         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 120         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 121         req = mux.SetURLVars(req, vars)
 122         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 123         response := executeRequest(req, handleFunc)
 124         checkResponseCode(t, http.StatusOK, response.Code)
 125         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 126         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 127         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 128                 t.Errorf("Incorrect alarm definition")
 129         }
 130 }
 131
 132 func TestDeleteAlarmDefinitions(t *testing.T) {
 133         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 134         //Get all
 135         var ricAlarmDefinitions RicAlarmDefinitions
 136         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 137         req = mux.SetURLVars(req, nil)
 138         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 139         response := executeRequest(req, handleFunc)
 140         checkResponseCode(t, http.StatusOK, response.Code)
 141         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 142         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 143                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 144         }
 145
 146         //Delete 8004
 147         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 148         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 149         req = mux.SetURLVars(req, vars)
 150         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 151         response = executeRequest(req, handleFunc)
 152         checkResponseCode(t, http.StatusOK, response.Code)
 153
 154         //Get 8004 fail
 155         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 156         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 157         req = mux.SetURLVars(req, vars)
 158         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 159         response = executeRequest(req, handleFunc)
 160         checkResponseCode(t, http.StatusBadRequest, response.Code)
 161
 162         //Set 8004 success
 163         var alarm8004Definition alarm.AlarmDefinition
 164         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 165         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 166         alarm8004Definition.EventType = "Processing error"
 167         alarm8004Definition.OperationInstructions = "Not defined"
 168         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 169         pbodyEn, _ := json.Marshal(pbodyParams)
 170         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 171         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 172         response = executeRequest(req, handleFunc)
 173         checkResponseCode(t, http.StatusOK, response.Code)
 174
 175         //Get 8004 success
 176         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 177         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 178         req = mux.SetURLVars(req, vars)
 179         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 180         response = executeRequest(req, handleFunc)
 181         checkResponseCode(t, http.StatusOK, response.Code)
 182 }
 183
 184 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 185         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 186         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 187         defer ts.Close()
 188
 189         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 190         assert.Nil(t, alarmer.Raise(a), "raise failed")
 191
 192         VerifyAlarm(t, a, 1)
 193 }
 194
 195 func TestAlarmClearedSucess(t *testing.T) {
 196         xapp.Logger.Info("TestAlarmClearedSucess")
 197         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 198         defer ts.Close()
 199
 200         // Raise the alarm
 201         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 202         assert.Nil(t, alarmer.Raise(a), "raise failed")
 203
 204         VerifyAlarm(t, a, 1)
 205
 206         // Now Clear the alarm and check alarm is removed
 207         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 208         assert.Nil(t, alarmer.Clear(a), "clear failed")
 209
 210         time.Sleep(time.Duration(2) * time.Second)
 211         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 212 }
 213
 214 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 215         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 216         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 217         defer ts.Close()
 218
 219         // Raise two alarms
 220         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 221         assert.Nil(t, alarmer.Raise(a), "raise failed")
 222
 223         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 224         assert.Nil(t, alarmer.Raise(b), "raise failed")
 225
 226         VerifyAlarm(t, a, 2)
 227         VerifyAlarm(t, b, 2)
 228 }
 229
 230 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 231         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 232         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 233         defer ts.Close()
 234
 235         // Raise two alarms
 236         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 237         assert.Nil(t, alarmer.Clear(a), "clear failed")
 238
 239         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 240         assert.Nil(t, alarmer.Clear(b), "clear failed")
 241
 242         time.Sleep(time.Duration(2) * time.Second)
 243         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 244 }
 245
 246 func TestAlarmsSuppresedSucess(t *testing.T) {
 247         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 248         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 249         defer ts.Close()
 250
 251         // Raise two similar/matching alarms ... the second one suppresed
 252         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 253         assert.Nil(t, alarmer.Raise(a), "raise failed")
 254         assert.Nil(t, alarmer.Raise(a), "raise failed")
 255
 256         VerifyAlarm(t, a, 1)
 257         assert.Nil(t, alarmer.Clear(a), "clear failed")
 258 }
 259
 260
 261 func TestInvalidAlarms(t *testing.T) {
 262         xapp.Logger.Info("TestInvalidAlarms")
 263         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 264         assert.Nil(t, alarmer.Raise(a), "raise failed")
 265         time.Sleep(time.Duration(2) * time.Second)
 266 }
 267
 268 func TestAlarmHandlingErrorCases(t *testing.T) {
 269         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 270         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 271         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 272         assert.Nil(t, ok, "raise failed")
 273 }
 274
 275 func TestConsumeUnknownMessage(t *testing.T) {
 276         xapp.Logger.Info("TestConsumeUnknownMessage")
 277         err := alarmManager.Consume(&xapp.RMRParams{})
 278         assert.Nil(t, err, "raise failed")
 279 }
 280
 281 func TestStatusCallback(t *testing.T) {
 282         xapp.Logger.Info("TestStatusCallback")
 283         assert.Equal(t, true, alarmManager.StatusCB())
 284 }
 285
 286 func TestActiveAlarmMaxThresholds(t *testing.T) {
 287         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 288         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 289         alarmManager.maxActiveAlarms = 0
 290         alarmManager.maxAlarmHistory = 10
 291
 292         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 293         assert.Nil(t, alarmer.Raise(a), "raise failed")
 294
 295         var alarmConfigParams alarm.AlarmConfigParams
 296         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 297         req = mux.SetURLVars(req, nil)
 298         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 299         response := executeRequest(req, handleFunc)
 300
 301         // Check HTTP Status Code
 302         checkResponseCode(t, http.StatusOK, response.Code)
 303
 304         // Decode the json output from handler
 305         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 306         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 307                 t.Errorf("Incorrect alarm thresholds")
 308         }
 309
 310         time.Sleep(time.Duration(1) * time.Second)
 311         alarmManager.maxActiveAlarms = 5000
 312         alarmManager.maxAlarmHistory = 20000
 313         VerifyAlarm(t, a, 2)
 314         VerifyAlarm(t, a, 2)
 315         ts.Close()
 316 }
 317
 318 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 319         receivedAlert := waitForEvent()
 320
 321         assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
 322         _, ok := alarmManager.IsMatchFound(a)
 323         assert.True(t, ok)
 324
 325         return receivedAlert
 326 }
 327
 328 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 329         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 330         //assert.Equal(t, receivedAlert, e)
 331 }
 332
 333 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 334         l, err := net.Listen("tcp", "localhost:9093")
 335         if err != nil {
 336                 t.Error("Failed to create listener: " + err.Error())
 337         }
 338         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 339                 assert.Equal(t, r.Method, method)
 340                 assert.Equal(t, r.URL.String(), url)
 341
 342                 fireEvent(t, r.Body)
 343
 344                 w.Header().Add("Content-Type", "application/json")
 345                 w.WriteHeader(status)
 346                 b, _ := json.Marshal(respData)
 347                 w.Write(b)
 348         }))
 349         ts.Listener.Close()
 350         ts.Listener = l
 351
 352         ts.Start()
 353
 354         return ts
 355 }
 356
 357 func waitForEvent() string {
 358         receivedAlert := <-eventChan
 359         return receivedAlert
 360 }
 361
 362 func fireEvent(t *testing.T, body io.ReadCloser) {
 363         reqBody, err := ioutil.ReadAll(body)
 364         assert.Nil(t, err, "ioutil.ReadAll failed")
 365         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 366
 367         eventChan <- fmt.Sprintf("%s", reqBody)
 368 }
 369
 370 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 371         rr := httptest.NewRecorder()
 372
 373         handleR.ServeHTTP(rr, req)
 374
 375         return rr
 376 }
 377
 378 func checkResponseCode(t *testing.T, expected, actual int) bool {
 379         if expected != actual {
 380                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 381                 return false
 382         }
 383         return true
 384 }