manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  28         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  29         "github.com/gorilla/mux"
  30         "github.com/prometheus/alertmanager/api/v2/models"
  31         "github.com/stretchr/testify/assert"
  32         "io"
  33         "io/ioutil"
  34         "net"
  35         "net/http"
  36         "net/http/httptest"
  37         "os"
  38         "strconv"
  39         "strings"
  40         "testing"
  41         "time"
  42 )
  43
  44 var alarmManager *AlarmManager
  45 var alarmer *alarm.RICAlarm
  46 var eventChan chan string
  47
  48 // Test cases
  49 func TestMain(M *testing.M) {
  50         os.Setenv("ALARM_IF_RMR", "true")
  51         alarmManager = NewAlarmManager("localhost:9093", 500)
  52         go alarmManager.Run(false)
  53         time.Sleep(time.Duration(10) * time.Second)
  54
  55         // Wait until RMR is up-and-running
  56         for !xapp.Rmr.IsReady() {
  57                 time.Sleep(time.Duration(1) * time.Second)
  58         }
  59
  60         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  61         alarmManager.alarmClient = alarmer
  62         time.Sleep(time.Duration(5) * time.Second)
  63         eventChan = make(chan string)
  64
  65         os.Exit(M.Run())
  66 }
  67
  68 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  69         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  70         var alarmDefinition alarm.AlarmDefinition
  71         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  72         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
  73         req = mux.SetURLVars(req, vars)
  74         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  75         response := executeRequest(req, handleFunc)
  76         checkResponseCode(t, http.StatusOK, response.Code)
  77         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  78         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  79         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
  80                 t.Errorf("Incorrect alarm definition")
  81         }
  82 }
  83
  84 func TestSetAlarmDefinitions(t *testing.T) {
  85         xapp.Logger.Info("TestSetAlarmDefinitions")
  86         var alarm8004Definition alarm.AlarmDefinition
  87         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  88         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  89         alarm8004Definition.EventType = "Processing error"
  90         alarm8004Definition.OperationInstructions = "Not defined"
  91
  92         var alarm8005Definition alarm.AlarmDefinition
  93         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  94         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  95         alarm8005Definition.EventType = "Communication error"
  96         alarm8005Definition.OperationInstructions = "Not defined"
  97
  98         var alarm8006Definition alarm.AlarmDefinition
  99         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
 100         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 101         alarm8006Definition.EventType = "Communication error"
 102         alarm8006Definition.OperationInstructions = "Not defined"
 103
 104         var alarm8007Definition alarm.AlarmDefinition
 105         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 106         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 107         alarm8007Definition.EventType = "Communication error"
 108         alarm8007Definition.OperationInstructions = "Not defined"
 109
 110         var alarm8008Definition alarm.AlarmDefinition
 111         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 112         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 113         alarm8008Definition.EventType = "storage warning"
 114         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
 115
 116         var alarm8009Definition alarm.AlarmDefinition
 117         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 118         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 119         alarm8009Definition.EventType = "storage warning"
 120         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
 121
 122         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 123         pbodyEn, _ := json.Marshal(pbodyParams)
 124         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 125         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 126         response := executeRequest(req, handleFunc)
 127         status := checkResponseCode(t, http.StatusOK, response.Code)
 128         xapp.Logger.Info("status = %v", status)
 129
 130 }
 131
 132 func TestGetAlarmDefinitions(t *testing.T) {
 133         xapp.Logger.Info("TestGetAlarmDefinitions")
 134         var alarmDefinition alarm.AlarmDefinition
 135         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 136         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 137         req = mux.SetURLVars(req, vars)
 138         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 139         response := executeRequest(req, handleFunc)
 140         checkResponseCode(t, http.StatusOK, response.Code)
 141         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 142         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 143         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 144                 t.Errorf("Incorrect alarm definition")
 145         }
 146 }
 147
 148 func TestDeleteAlarmDefinitions(t *testing.T) {
 149         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 150         //Get all
 151         var ricAlarmDefinitions RicAlarmDefinitions
 152         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 153         req = mux.SetURLVars(req, nil)
 154         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 155         response := executeRequest(req, handleFunc)
 156         checkResponseCode(t, http.StatusOK, response.Code)
 157         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 158         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 159                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 160         }
 161
 162         //Delete 8004
 163         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 164         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 165         req = mux.SetURLVars(req, vars)
 166         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 167         response = executeRequest(req, handleFunc)
 168         checkResponseCode(t, http.StatusOK, response.Code)
 169
 170         //Get 8004 fail
 171         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 172         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 173         req = mux.SetURLVars(req, vars)
 174         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 175         response = executeRequest(req, handleFunc)
 176         checkResponseCode(t, http.StatusBadRequest, response.Code)
 177
 178         //Set 8004 success
 179         var alarm8004Definition alarm.AlarmDefinition
 180         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 181         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 182         alarm8004Definition.EventType = "Processing error"
 183         alarm8004Definition.OperationInstructions = "Not defined"
 184         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 185         pbodyEn, _ := json.Marshal(pbodyParams)
 186         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 187         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 188         response = executeRequest(req, handleFunc)
 189         checkResponseCode(t, http.StatusOK, response.Code)
 190
 191         //Get 8004 success
 192         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 193         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 194         req = mux.SetURLVars(req, vars)
 195         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 196         response = executeRequest(req, handleFunc)
 197         checkResponseCode(t, http.StatusOK, response.Code)
 198 }
 199
 200 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 201         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 202         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 203         defer ts.Close()
 204
 205         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 206         assert.Nil(t, alarmer.Raise(a), "raise failed")
 207
 208         VerifyAlarm(t, a, 1)
 209 }
 210
 211 func TestAlarmClearedSucess(t *testing.T) {
 212         xapp.Logger.Info("TestAlarmClearedSucess")
 213         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 214         defer ts.Close()
 215
 216         // Raise the alarm
 217         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 218         assert.Nil(t, alarmer.Raise(a), "raise failed")
 219
 220         VerifyAlarm(t, a, 1)
 221
 222         // Now Clear the alarm and check alarm is removed
 223         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 224         assert.Nil(t, alarmer.Clear(a), "clear failed")
 225
 226         time.Sleep(time.Duration(2) * time.Second)
 227         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 228 }
 229
 230 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 231         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 232         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 233         defer ts.Close()
 234
 235         // Raise two alarms
 236         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 237         assert.Nil(t, alarmer.Raise(a), "raise failed")
 238
 239         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 240         assert.Nil(t, alarmer.Raise(b), "raise failed")
 241
 242         VerifyAlarm(t, a, 2)
 243         VerifyAlarm(t, b, 2)
 244 }
 245
 246 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 247         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 248         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 249         defer ts.Close()
 250
 251         // Raise two alarms
 252         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 253         assert.Nil(t, alarmer.Clear(a), "clear failed")
 254
 255         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 256         assert.Nil(t, alarmer.Clear(b), "clear failed")
 257
 258         time.Sleep(time.Duration(2) * time.Second)
 259         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 260 }
 261
 262 func TestAlarmsSuppresedSucess(t *testing.T) {
 263         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 264         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 265         defer ts.Close()
 266
 267         // Raise two similar/matching alarms ... the second one suppresed
 268         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 269         assert.Nil(t, alarmer.Raise(a), "raise failed")
 270         assert.Nil(t, alarmer.Raise(a), "raise failed")
 271
 272         VerifyAlarm(t, a, 1)
 273         assert.Nil(t, alarmer.Clear(a), "clear failed")
 274 }
 275
 276 func TestInvalidAlarms(t *testing.T) {
 277         xapp.Logger.Info("TestInvalidAlarms")
 278         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 279         assert.Nil(t, alarmer.Raise(a), "raise failed")
 280         time.Sleep(time.Duration(2) * time.Second)
 281 }
 282
 283 func TestAlarmHandlingErrorCases(t *testing.T) {
 284         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 285         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 286         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 287         assert.Nil(t, ok, "raise failed")
 288 }
 289
 290 func TestConsumeUnknownMessage(t *testing.T) {
 291         xapp.Logger.Info("TestConsumeUnknownMessage")
 292         err := alarmManager.Consume(&xapp.RMRParams{})
 293         assert.Nil(t, err, "raise failed")
 294 }
 295
 296 func TestStatusCallback(t *testing.T) {
 297         xapp.Logger.Info("TestStatusCallback")
 298         assert.Equal(t, true, alarmManager.StatusCB())
 299 }
 300
 301 func TestActiveAlarmMaxThresholds(t *testing.T) {
 302         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 303         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 304         alarmManager.maxActiveAlarms = 0
 305         alarmManager.maxAlarmHistory = 10
 306
 307         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 308         assert.Nil(t, alarmer.Raise(a), "raise failed")
 309
 310         var alarmConfigParams alarm.AlarmConfigParams
 311         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 312         req = mux.SetURLVars(req, nil)
 313         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 314         response := executeRequest(req, handleFunc)
 315
 316         // Check HTTP Status Code
 317         checkResponseCode(t, http.StatusOK, response.Code)
 318
 319         // Decode the json output from handler
 320         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 321         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 322                 t.Errorf("Incorrect alarm thresholds")
 323         }
 324
 325         time.Sleep(time.Duration(1) * time.Second)
 326         alarmManager.maxActiveAlarms = 5000
 327         alarmManager.maxAlarmHistory = 20000
 328         VerifyAlarm(t, a, 2)
 329         VerifyAlarm(t, a, 2)
 330         ts.Close()
 331 }
 332
 333 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 334         receivedAlert := waitForEvent()
 335
 336         assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
 337         _, ok := alarmManager.IsMatchFound(a)
 338         assert.True(t, ok)
 339
 340         return receivedAlert
 341 }
 342
 343 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 344         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 345         //assert.Equal(t, receivedAlert, e)
 346 }
 347
 348 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 349         l, err := net.Listen("tcp", "localhost:9093")
 350         if err != nil {
 351                 t.Error("Failed to create listener: " + err.Error())
 352         }
 353         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 354                 assert.Equal(t, r.Method, method)
 355                 assert.Equal(t, r.URL.String(), url)
 356
 357                 fireEvent(t, r.Body)
 358
 359                 w.Header().Add("Content-Type", "application/json")
 360                 w.WriteHeader(status)
 361                 b, _ := json.Marshal(respData)
 362                 w.Write(b)
 363         }))
 364         ts.Listener.Close()
 365         ts.Listener = l
 366
 367         ts.Start()
 368
 369         return ts
 370 }
 371
 372 func waitForEvent() string {
 373         receivedAlert := <-eventChan
 374         return receivedAlert
 375 }
 376
 377 func fireEvent(t *testing.T, body io.ReadCloser) {
 378         reqBody, err := ioutil.ReadAll(body)
 379         assert.Nil(t, err, "ioutil.ReadAll failed")
 380         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 381
 382         eventChan <- fmt.Sprintf("%s", reqBody)
 383 }
 384
 385 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 386         rr := httptest.NewRecorder()
 387
 388         handleR.ServeHTTP(rr, req)
 389
 390         return rr
 391 }
 392
 393 func checkResponseCode(t *testing.T, expected, actual int) bool {
 394         if expected != actual {
 395                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 396                 return false
 397         }
 398         return true
 399 }