manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  28         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  29         "github.com/gorilla/mux"
  30         "github.com/prometheus/alertmanager/api/v2/models"
  31         "github.com/stretchr/testify/assert"
  32         "io"
  33         "io/ioutil"
  34         "net"
  35         "net/http"
  36         "net/http/httptest"
  37         "os"
  38         "os/exec"
  39         "strconv"
  40         "strings"
  41         "testing"
  42         "time"
  43 )
  44
  45 var alarmManager *AlarmManager
  46 var alarmer *alarm.RICAlarm
  47 var eventChan chan string
  48
  49 // Test cases
  50 func TestMain(M *testing.M) {
  51         alarmManager = NewAlarmManager("localhost:9093", 500)
  52         alarmManager.alertInterval = 20000
  53         go alarmManager.Run(false)
  54         time.Sleep(time.Duration(10) * time.Second)
  55
  56         // Wait until RMR is up-and-running
  57         for !xapp.Rmr.IsReady() {
  58                 time.Sleep(time.Duration(1) * time.Second)
  59         }
  60
  61         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  62         alarmManager.alarmClient = alarmer
  63         time.Sleep(time.Duration(5) * time.Second)
  64         eventChan = make(chan string)
  65
  66         os.Exit(M.Run())
  67 }
  68
  69 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  70         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  71         var alarmDefinition alarm.AlarmDefinition
  72         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  73         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
  74         req = mux.SetURLVars(req, vars)
  75         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  76         response := executeRequest(req, handleFunc)
  77         checkResponseCode(t, http.StatusOK, response.Code)
  78         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  79         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  80         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
  81                 t.Errorf("Incorrect alarm definition")
  82         }
  83 }
  84
  85 func TestSetAlarmDefinitions(t *testing.T) {
  86         xapp.Logger.Info("TestSetAlarmDefinitions")
  87         var alarm8004Definition alarm.AlarmDefinition
  88         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  89         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  90         alarm8004Definition.EventType = "Processing error"
  91         alarm8004Definition.OperationInstructions = "Not defined"
  92
  93         var alarm8005Definition alarm.AlarmDefinition
  94         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  95         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  96         alarm8005Definition.EventType = "Communication error"
  97         alarm8005Definition.OperationInstructions = "Not defined"
  98
  99         var alarm8006Definition alarm.AlarmDefinition
 100         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
 101         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 102         alarm8006Definition.EventType = "Communication error"
 103         alarm8006Definition.OperationInstructions = "Not defined"
 104
 105         var alarm8007Definition alarm.AlarmDefinition
 106         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 107         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 108         alarm8007Definition.EventType = "Communication error"
 109         alarm8007Definition.OperationInstructions = "Not defined"
 110
 111         var alarm8008Definition alarm.AlarmDefinition
 112         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 113         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 114         alarm8008Definition.EventType = "storage warning"
 115         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
 116
 117         var alarm8009Definition alarm.AlarmDefinition
 118         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 119         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 120         alarm8009Definition.EventType = "storage warning"
 121         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
 122
 123         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 124         pbodyEn, _ := json.Marshal(pbodyParams)
 125         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 126         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 127         response := executeRequest(req, handleFunc)
 128         status := checkResponseCode(t, http.StatusOK, response.Code)
 129         xapp.Logger.Info("status = %v", status)
 130
 131 }
 132
 133 func TestGetAlarmDefinitions(t *testing.T) {
 134         xapp.Logger.Info("TestGetAlarmDefinitions")
 135         var alarmDefinition alarm.AlarmDefinition
 136         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 137         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 138         req = mux.SetURLVars(req, vars)
 139         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 140         response := executeRequest(req, handleFunc)
 141         checkResponseCode(t, http.StatusOK, response.Code)
 142         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 143         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 144         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 145                 t.Errorf("Incorrect alarm definition")
 146         }
 147 }
 148
 149 func TestDeleteAlarmDefinitions(t *testing.T) {
 150         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 151         //Get all
 152         var ricAlarmDefinitions RicAlarmDefinitions
 153         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 154         req = mux.SetURLVars(req, nil)
 155         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 156         response := executeRequest(req, handleFunc)
 157         checkResponseCode(t, http.StatusOK, response.Code)
 158         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 159         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 160                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 161         }
 162
 163         //Delete 8004
 164         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 165         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 166         req = mux.SetURLVars(req, vars)
 167         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 168         response = executeRequest(req, handleFunc)
 169         checkResponseCode(t, http.StatusOK, response.Code)
 170
 171         //Get 8004 fail
 172         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 173         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 174         req = mux.SetURLVars(req, vars)
 175         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 176         response = executeRequest(req, handleFunc)
 177         checkResponseCode(t, http.StatusBadRequest, response.Code)
 178
 179         //Set 8004 success
 180         var alarm8004Definition alarm.AlarmDefinition
 181         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 182         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 183         alarm8004Definition.EventType = "Processing error"
 184         alarm8004Definition.OperationInstructions = "Not defined"
 185         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 186         pbodyEn, _ := json.Marshal(pbodyParams)
 187         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 188         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 189         response = executeRequest(req, handleFunc)
 190         checkResponseCode(t, http.StatusOK, response.Code)
 191
 192         //Get 8004 success
 193         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 194         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 195         req = mux.SetURLVars(req, vars)
 196         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 197         response = executeRequest(req, handleFunc)
 198         checkResponseCode(t, http.StatusOK, response.Code)
 199 }
 200
 201 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 202         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 203         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 204         defer ts.Close()
 205
 206         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 207         assert.Nil(t, alarmer.Raise(a), "raise failed")
 208
 209         VerifyAlarm(t, a, 1)
 210 }
 211
 212 func TestAlarmClearedSucess(t *testing.T) {
 213         xapp.Logger.Info("TestAlarmClearedSucess")
 214         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 215         defer ts.Close()
 216
 217         // Raise the alarm
 218         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 219         assert.Nil(t, alarmer.Raise(a), "raise failed")
 220
 221         VerifyAlarm(t, a, 1)
 222
 223         // Now Clear the alarm and check alarm is removed
 224         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 225         assert.Nil(t, alarmer.Clear(a), "clear failed")
 226
 227         time.Sleep(time.Duration(2) * time.Second)
 228         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 229 }
 230
 231 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 232         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 233         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 234         defer ts.Close()
 235
 236         // Raise two alarms
 237         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 238         assert.Nil(t, alarmer.Raise(a), "raise failed")
 239
 240         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 241         assert.Nil(t, alarmer.Raise(b), "raise failed")
 242
 243         time.Sleep(time.Duration(2) * time.Second)
 244         VerifyAlarm(t, a, 2)
 245         VerifyAlarm(t, b, 2)
 246 }
 247
 248 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 249         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 250         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 251         defer ts.Close()
 252
 253         // Raise two alarms
 254         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 255         assert.Nil(t, alarmer.Clear(a), "clear failed")
 256
 257         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 258         assert.Nil(t, alarmer.Clear(b), "clear failed")
 259
 260         time.Sleep(time.Duration(2) * time.Second)
 261         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 262 }
 263
 264 func TestAlarmsSuppresedSucess(t *testing.T) {
 265         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 266         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 267         defer ts.Close()
 268
 269         // Raise two similar/matching alarms ... the second one suppresed
 270         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 271         assert.Nil(t, alarmer.Raise(a), "raise failed")
 272         assert.Nil(t, alarmer.Raise(a), "raise failed")
 273
 274         VerifyAlarm(t, a, 1)
 275         assert.Nil(t, alarmer.Clear(a), "clear failed")
 276 }
 277
 278 func TestInvalidAlarms(t *testing.T) {
 279         xapp.Logger.Info("TestInvalidAlarms")
 280         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 281         assert.Nil(t, alarmer.Raise(a), "raise failed")
 282         time.Sleep(time.Duration(2) * time.Second)
 283 }
 284
 285 func TestAlarmHandlingErrorCases(t *testing.T) {
 286         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 287         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 288         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 289         assert.Nil(t, ok, "raise failed")
 290 }
 291
 292 func TestConsumeUnknownMessage(t *testing.T) {
 293         xapp.Logger.Info("TestConsumeUnknownMessage")
 294         err := alarmManager.Consume(&xapp.RMRParams{})
 295         assert.Nil(t, err, "raise failed")
 296 }
 297
 298 func TestStatusCallback(t *testing.T) {
 299         xapp.Logger.Info("TestStatusCallback")
 300         assert.Equal(t, true, alarmManager.StatusCB())
 301 }
 302
 303 func TestActiveAlarmMaxThresholds(t *testing.T) {
 304         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 305         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 306         alarmManager.maxActiveAlarms = 0
 307         alarmManager.maxAlarmHistory = 10
 308
 309         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 310         assert.Nil(t, alarmer.Raise(a), "raise failed")
 311
 312         var alarmConfigParams alarm.AlarmConfigParams
 313         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 314         req = mux.SetURLVars(req, nil)
 315         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 316         response := executeRequest(req, handleFunc)
 317
 318         // Check HTTP Status Code
 319         checkResponseCode(t, http.StatusOK, response.Code)
 320
 321         // Decode the json output from handler
 322         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 323         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 324                 t.Errorf("Incorrect alarm thresholds")
 325         }
 326
 327         time.Sleep(time.Duration(1) * time.Second)
 328         alarmManager.maxActiveAlarms = 5000
 329         alarmManager.maxAlarmHistory = 20000
 330         VerifyAlarm(t, a, 2)
 331         VerifyAlarm(t, a, 2)
 332         ts.Close()
 333 }
 334
 335 func TestGetPrometheusAlerts(t *testing.T) {
 336         time.Sleep(1 * time.Second)
 337         xapp.Logger.Info("TestGetPrometheusAlerts")
 338         ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
 339
 340         commandReady := make(chan bool, 1)
 341         command := "cli/alarm-cli"
 342         args := []string{"gapam", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
 343         ExecCLICommand(commandReady, command, args...)
 344         <-commandReady
 345
 346         ts.Close()
 347 }
 348
 349 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 350         receivedAlert := waitForEvent()
 351
 352         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 353         _, ok := alarmManager.IsMatchFound(a)
 354         assert.True(t, ok)
 355
 356         return receivedAlert
 357 }
 358
 359 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 360         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 361         //assert.Equal(t, receivedAlert, e)
 362 }
 363
 364 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 365         l, err := net.Listen("tcp", "localhost:9093")
 366         if err != nil {
 367                 t.Error("Failed to create listener: " + err.Error())
 368         }
 369         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 370                 assert.Equal(t, r.Method, method)
 371                 assert.Equal(t, r.URL.String(), url)
 372
 373                 fireEvent(t, r.Body)
 374
 375                 w.Header().Add("Content-Type", "application/json")
 376                 w.WriteHeader(status)
 377                 b, _ := json.Marshal(respData)
 378                 w.Write(b)
 379         }))
 380         ts.Listener.Close()
 381         ts.Listener = l
 382
 383         ts.Start()
 384
 385         return ts
 386 }
 387
 388 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
 389         l, err := net.Listen("tcp", "localhost:9093")
 390         if err != nil {
 391                 t.Error("Failed to create listener: " + err.Error())
 392         }
 393         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 394                 assert.Equal(t, r.Method, method)
 395                 assert.Equal(t, r.URL.String(), url)
 396
 397                 w.Header().Add("Content-Type", "application/json")
 398                 w.WriteHeader(200)
 399                 // Read alerts from file
 400                 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
 401                 if err != nil {
 402                         t.Error("Failed to send response: ", err)
 403                 }
 404                 _, err = w.Write(payload)
 405                 if err != nil {
 406                         t.Error("Failed to send response: " + err.Error())
 407                 }
 408         }))
 409         ts.Listener.Close()
 410         ts.Listener = l
 411         ts.Start()
 412         return ts
 413 }
 414
 415 func waitForEvent() string {
 416         receivedAlert := <-eventChan
 417         return receivedAlert
 418 }
 419
 420 func fireEvent(t *testing.T, body io.ReadCloser) {
 421         reqBody, err := ioutil.ReadAll(body)
 422         assert.Nil(t, err, "ioutil.ReadAll failed")
 423         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 424
 425         eventChan <- fmt.Sprintf("%s", reqBody)
 426 }
 427
 428 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 429         rr := httptest.NewRecorder()
 430
 431         handleR.ServeHTTP(rr, req)
 432
 433         return rr
 434 }
 435
 436 func checkResponseCode(t *testing.T, expected, actual int) bool {
 437         if expected != actual {
 438                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 439                 return false
 440         }
 441         return true
 442 }
 443
 444 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
 445         go func() {
 446                 xapp.Logger.Info("Giving CLI command")
 447                 cmd := exec.Command(command, args...)
 448                 cmd.Dir = "../"
 449                 output, err := cmd.CombinedOutput()
 450                 if err != nil {
 451                         xapp.Logger.Info("CLI command failed out: %s", err)
 452                 }
 453                 xapp.Logger.Info("CLI command output: %s", output)
 454                 commandReady <- true
 455                 xapp.Logger.Info("CLI command completed")
 456         }()
 457 }
 458
 459 func readJSONFromFile(filename string) ([]byte, error) {
 460         file, err := ioutil.ReadFile(filename)
 461         if err != nil {
 462                 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
 463                 return nil, err
 464         }
 465         return file, nil
 466 }