manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io"
  28         "io/ioutil"
  29         "net"
  30         "net/http"
  31         "net/http/httptest"
  32         "os"
  33         "os/exec"
  34         "strconv"
  35         "strings"
  36         "testing"
  37         "time"
  38
  39         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  40         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  41         "github.com/gorilla/mux"
  42         "github.com/prometheus/alertmanager/api/v2/models"
  43         "github.com/stretchr/testify/assert"
  44 )
  45
  46 var alarmManager *AlarmManager
  47 var alarmer *alarm.RICAlarm
  48 var eventChan chan string
  49
  50 // Test cases
  51 func TestMain(M *testing.M) {
  52         alarmManager = NewAlarmManager("localhost:9093", 500, false)
  53         alarmManager.alertInterval = 20000
  54         go alarmManager.Run(false, 5)
  55         time.Sleep(time.Duration(10) * time.Second)
  56
  57         // Wait until RMR is up-and-running
  58         for !xapp.Rmr.IsReady() {
  59                 time.Sleep(time.Duration(1) * time.Second)
  60         }
  61
  62         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  63         alarmManager.alarmClient = alarmer
  64         time.Sleep(time.Duration(5) * time.Second)
  65         eventChan = make(chan string)
  66
  67         os.Exit(M.Run())
  68 }
  69
  70 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  71         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  72         var alarmDefinition alarm.AlarmDefinition
  73         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  74         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
  75         req = mux.SetURLVars(req, vars)
  76         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  77         response := executeRequest(req, handleFunc)
  78         checkResponseCode(t, http.StatusOK, response.Code)
  79         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  80         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  81     if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" {
  82                 t.Errorf("Incorrect alarm definition")
  83         }
  84 }
  85
  86 func TestSetAlarmDefinitions(t *testing.T) {
  87         xapp.Logger.Info("TestSetAlarmDefinitions")
  88
  89     var alarm72004Definition alarm.AlarmDefinition
  90         alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM
  91         alarm72004Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB/G-NODEB"
  92         alarm72004Definition.EventType = "Communication error"
  93         alarm72004Definition.OperationInstructions = "Not defined"
  94         alarm72004Definition.RaiseDelay = 0
  95         alarm72004Definition.ClearDelay = 0
  96
  97         var alarm72008Definition alarm.AlarmDefinition
  98         alarm72008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
  99         alarm72008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 100         alarm72008Definition.EventType = "storage warning"
 101         alarm72008Definition.OperationInstructions = "Clear alarms or raise threshold"
 102         alarm72008Definition.RaiseDelay = 0
 103         alarm72008Definition.ClearDelay = 0
 104
 105         var alarm72009Definition alarm.AlarmDefinition
 106         alarm72009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 107         alarm72009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 108         alarm72009Definition.EventType = "storage warning"
 109         alarm72009Definition.OperationInstructions = "Clear alarms or raise threshold"
 110         alarm72009Definition.RaiseDelay = 0
 111         alarm72009Definition.ClearDelay = 0
 112
 113         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition, &alarm72008Definition, &alarm72009Definition}}
 114         pbodyEn, _ := json.Marshal(pbodyParams)
 115         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 116         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 117         response := executeRequest(req, handleFunc)
 118         status := checkResponseCode(t, http.StatusOK, response.Code)
 119         xapp.Logger.Info("status = %v", status)
 120 }
 121
 122
 123 func TestSetAlarmConfigDecodeError(t *testing.T) {
 124         xapp.Logger.Info("TestSetAlarmConfigDecodeError")
 125
 126         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 127         req, _ := http.NewRequest("POST", "/ric/v1/alarms", bytes.NewBuffer(jsonStr))
 128         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 129         response := executeRequest(req, handleFunc)
 130         status := checkResponseCode(t,http.StatusOK,response.Code)
 131         xapp.Logger.Info("status = %v", status)
 132 }
 133
 134 func TestSetAlarmDefinitionDecodeError(t *testing.T) {
 135         xapp.Logger.Info("TestSetAlarmDefinitionDecodeError")
 136
 137         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 138         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(jsonStr))
 139         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 140         response := executeRequest(req, handleFunc)
 141         status := checkResponseCode(t,http.StatusBadRequest,response.Code)
 142         xapp.Logger.Info("status = %v", status)
 143 }
 144
 145 func TestRaiseAlarmEmptyBody(t *testing.T) {
 146         xapp.Logger.Info("TestRaiseAlarmEmptyBody")
 147         req, _ := http.NewRequest("POST", "/ric/v1/alarms", nil)
 148         handleFunc := http.HandlerFunc(alarmManager.RaiseAlarm)
 149         response := executeRequest(req, handleFunc)
 150         status := checkResponseCode(t,http.StatusOK,response.Code)
 151         xapp.Logger.Info("status = %v", status)
 152 }
 153
 154 func TestSetAlarmDefinitionsEmptyBody(t *testing.T) {
 155         xapp.Logger.Info("TestSetAlarmDefinitionsEmptyBody")
 156         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", nil)
 157         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 158         response := executeRequest(req, handleFunc)
 159         status := checkResponseCode(t,http.StatusBadRequest,response.Code)
 160         xapp.Logger.Info("status = %v", status)
 161 }
 162
 163 func TestClearAlarmEmptyBody(t *testing.T) {
 164         xapp.Logger.Info("TestClearAlarmEmptyBody")
 165         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms", nil)
 166         handleFunc := http.HandlerFunc(alarmManager.ClearAlarm)
 167         response := executeRequest(req, handleFunc)
 168         status := checkResponseCode(t,http.StatusOK,response.Code)
 169         xapp.Logger.Info("status = %v", status)
 170 }
 171
 172 func TestGetAlarmDefinitions(t *testing.T) {
 173         xapp.Logger.Info("TestGetAlarmDefinitions")
 174         var alarmDefinition alarm.AlarmDefinition
 175         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 176         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 177         req = mux.SetURLVars(req, vars)
 178         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 179         response := executeRequest(req, handleFunc)
 180         checkResponseCode(t, http.StatusOK, response.Code)
 181         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 182         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 183         if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" {
 184                 t.Errorf("Incorrect alarm definition")
 185         }
 186 }
 187
 188 func TestDeleteAlarmDefinitions(t *testing.T) {
 189         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 190         //Get all
 191         var ricAlarmDefinitions RicAlarmDefinitions
 192         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 193         req = mux.SetURLVars(req, nil)
 194         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 195         response := executeRequest(req, handleFunc)
 196         checkResponseCode(t, http.StatusOK, response.Code)
 197         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 198         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 199                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 200         }
 201
 202         //Delete 72004
 203         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 204         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 205         req = mux.SetURLVars(req, vars)
 206         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 207         response = executeRequest(req, handleFunc)
 208         checkResponseCode(t, http.StatusOK, response.Code)
 209
 210         //Get 72004 fail
 211         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 212         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 213         req = mux.SetURLVars(req, vars)
 214         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 215         response = executeRequest(req, handleFunc)
 216         checkResponseCode(t, http.StatusBadRequest, response.Code)
 217
 218         //Delete Alarm which doesn't present
 219         //Set 72004 success
 220         var alarm72004Definition alarm.AlarmDefinition
 221         alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM
 222         alarm72004Definition.AlarmText = "E2 CONNECTION PROBLEM"
 223         alarm72004Definition.EventType = "Processing error"
 224         alarm72004Definition.OperationInstructions = "Not defined"
 225         alarm72004Definition.RaiseDelay = 0
 226         alarm72004Definition.ClearDelay = 0
 227         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition}}
 228         pbodyEn, _ := json.Marshal(pbodyParams)
 229         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 230         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 231         response = executeRequest(req, handleFunc)
 232         checkResponseCode(t, http.StatusOK, response.Code)
 233
 234         //Get 72004 success
 235         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 236         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 237         req = mux.SetURLVars(req, vars)
 238         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 239         response = executeRequest(req, handleFunc)
 240         checkResponseCode(t, http.StatusOK, response.Code)
 241 }
 242
 243 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 244         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 245         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 246         defer ts.Close()
 247
 248         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 249         assert.Nil(t, alarmer.Raise(a), "raise failed")
 250
 251         VerifyAlarm(t, a, 1)
 252
 253         var activeAlarms []AlarmNotification
 254         activeAlarms = make([]AlarmNotification, 1)
 255         req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil)
 256         req = mux.SetURLVars(req, nil)
 257         handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms)
 258         response := executeRequest(req, handleFunc)
 259         checkResponseCode(t, http.StatusOK, response.Code)
 260
 261         // Decode the json output from handler
 262         json.NewDecoder(response.Body).Decode(activeAlarms)
 263         if len(activeAlarms) != 1 {
 264                 t.Errorf("Incorrect alarm alarm count")
 265         }
 266
 267         var alarmHistory []AlarmNotification
 268         alarmHistory = make([]AlarmNotification, 1)
 269         req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil)
 270         req = mux.SetURLVars(req, nil)
 271         handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory)
 272         response = executeRequest(req, handleFunc)
 273         checkResponseCode(t, http.StatusOK, response.Code)
 274
 275         // Decode the json output from handler
 276         json.NewDecoder(response.Body).Decode(alarmHistory)
 277         if len(alarmHistory) != 1 {
 278                 t.Errorf("Incorrect alarm history count")
 279         }
 280 }
 281
 282 func TestAlarmClearedSucess(t *testing.T) {
 283         xapp.Logger.Info("TestAlarmClearedSucess")
 284         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 285         defer ts.Close()
 286
 287         // Raise the alarm
 288         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 289         assert.Nil(t, alarmer.Raise(a), "raise failed")
 290
 291         VerifyAlarm(t, a, 1)
 292
 293         // Now Clear the alarm and check alarm is removed
 294         a = alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 295     assert.Nil(t, alarmer.Clear(a), "clear failed")
 296
 297         time.Sleep(time.Duration(2) * time.Second)
 298         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 299 }
 300
 301 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 302         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 303         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 304         defer ts.Close()
 305
 306         // Raise two alarms
 307         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 308     assert.Nil(t, alarmer.Raise(a), "raise failed")
 309
 310         b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11")
 311         assert.Nil(t, alarmer.Raise(b), "raise failed")
 312
 313         time.Sleep(time.Duration(5) * time.Second)
 314
 315         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 316         VerifyAlarm(t, a, 1)
 317         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 318         VerifyAlarm(t, b, 2)
 319 }
 320
 321 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 322         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 323         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 324         defer ts.Close()
 325
 326         // Raise two alarms
 327         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 328         assert.Nil(t, alarmer.Clear(a), "clear failed")
 329
 330         b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11")
 331         assert.Nil(t, alarmer.Clear(b), "clear failed")
 332
 333         time.Sleep(time.Duration(2) * time.Second)
 334         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 335 }
 336
 337 func TestAlarmsSuppresedSucess(t *testing.T) {
 338         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 339         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 340         defer ts.Close()
 341
 342         // Raise two similar/matching alarms ... the second one suppresed
 343         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 344         assert.Nil(t, alarmer.Raise(a), "raise failed")
 345         assert.Nil(t, alarmer.Raise(a), "raise failed")
 346
 347         VerifyAlarm(t, a, 1)
 348         assert.Nil(t, alarmer.Clear(a), "clear failed")
 349 }
 350
 351 func TestInvalidAlarms(t *testing.T) {
 352         xapp.Logger.Info("TestInvalidAlarms")
 353         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 354         assert.Nil(t, alarmer.Raise(a), "raise failed")
 355         time.Sleep(time.Duration(2) * time.Second)
 356 }
 357
 358 func TestAlarmHandlingErrorCases(t *testing.T) {
 359         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 360         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 361         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 362         assert.Nil(t, ok, "raise failed")
 363 }
 364
 365 func TestConsumeUnknownMessage(t *testing.T) {
 366         xapp.Logger.Info("TestConsumeUnknownMessage")
 367         err := alarmManager.Consume(&xapp.RMRParams{})
 368         assert.Nil(t, err, "raise failed")
 369 }
 370
 371 func TestStatusCallback(t *testing.T) {
 372         xapp.Logger.Info("TestStatusCallback")
 373         assert.Equal(t, true, alarmManager.StatusCB())
 374 }
 375
 376 func TestActiveAlarmMaxThresholds(t *testing.T) {
 377         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 378         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 379         alarmManager.maxActiveAlarms = 0
 380         alarmManager.maxAlarmHistory = 10
 381
 382         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 383         assert.Nil(t, alarmer.Raise(a), "raise failed")
 384
 385         var alarmConfigParams alarm.AlarmConfigParams
 386         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 387         req = mux.SetURLVars(req, nil)
 388         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 389         response := executeRequest(req, handleFunc)
 390
 391         // Check HTTP Status Code
 392         checkResponseCode(t, http.StatusOK, response.Code)
 393
 394         // Decode the json output from handler
 395         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 396         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 397                 t.Errorf("Incorrect alarm thresholds")
 398         }
 399
 400         time.Sleep(time.Duration(1) * time.Second)
 401         alarmManager.maxActiveAlarms = 5000
 402         alarmManager.maxAlarmHistory = 20000
 403         VerifyAlarm(t, a, 2)
 404         VerifyAlarm(t, a, 2)
 405         ts.Close()
 406 }
 407
 408 func TestGetPrometheusAlerts(t *testing.T) {
 409         time.Sleep(1 * time.Second)
 410         xapp.Logger.Info("TestGetPrometheusAlerts")
 411         ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
 412
 413         commandReady := make(chan bool, 1)
 414         command := "cli/alarm-cli"
 415         args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
 416         ExecCLICommand(commandReady, command, args...)
 417         <-commandReady
 418
 419         ts.Close()
 420 }
 421
 422 func TestDelayedAlarmRaiseAndClear(t *testing.T) {
 423         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
 424
 425         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 426         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 427
 428         // Add new alarm definition
 429         var alarm9999Definition alarm.AlarmDefinition
 430         alarm9999Definition.AlarmId = 9999
 431         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 432         alarm9999Definition.EventType = "Test type"
 433         alarm9999Definition.OperationInstructions = "Not defined"
 434         alarm9999Definition.RaiseDelay = 1
 435         alarm9999Definition.ClearDelay = 1
 436         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 437         pbodyEn, _ := json.Marshal(pbodyParams)
 438         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 439         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 440         response := executeRequest(req, handleFunc)
 441         checkResponseCode(t, http.StatusOK, response.Code)
 442
 443         // Verify 9999 alarm definition
 444         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 445         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 446         req = mux.SetURLVars(req, vars)
 447         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 448         response = executeRequest(req, handleFunc)
 449         checkResponseCode(t, http.StatusOK, response.Code)
 450
 451         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 452         defer ts.Close()
 453
 454         // Raise alarm. Posting alert and updating alarm history should be delayed
 455         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 456         assert.Nil(t, alarmer.Raise(a), "raise failed")
 457         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 458
 459         // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
 460         assert.Nil(t, alarmer.Clear(a), "clear failed")
 461
 462         time.Sleep(time.Duration(2) * time.Second)
 463         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 464         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
 465 }
 466
 467 func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
 468         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
 469
 470         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 471         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 472
 473         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 474         defer ts.Close()
 475
 476         // Raise two alarms. The first should be delayed
 477         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 478         assert.Nil(t, alarmer.Raise(a), "raise failed")
 479         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 480
 481         b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 482         assert.Nil(t, alarmer.Raise(b), "raise failed")
 483         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 484
 485         // Clear two alarms. The first should be delayed. Check the alarms are removed
 486         assert.Nil(t, alarmer.Clear(a), "clear failed")
 487         assert.Nil(t, alarmer.Clear(b), "clear failed")
 488
 489         time.Sleep(time.Duration(2) * time.Second)
 490         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 491         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 492 }
 493
 494 func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
 495         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
 496
 497         // Delete exisitng 9999 alarm definition
 498         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 499         vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
 500         req = mux.SetURLVars(req, vars)
 501         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 502         response := executeRequest(req, handleFunc)
 503         checkResponseCode(t, http.StatusOK, response.Code)
 504
 505         // Add updated 9999 alarm definition
 506         var alarm9999Definition alarm.AlarmDefinition
 507         alarm9999Definition.AlarmId = 9999
 508         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 509         alarm9999Definition.EventType = "Test type"
 510         alarm9999Definition.OperationInstructions = "Not defined"
 511         alarm9999Definition.RaiseDelay = 1
 512         alarm9999Definition.ClearDelay = 0
 513         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 514         pbodyEn, _ := json.Marshal(pbodyParams)
 515         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 516         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 517         response = executeRequest(req, handleFunc)
 518         checkResponseCode(t, http.StatusOK, response.Code)
 519
 520         // Verify 9999 alarm definition
 521         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 522         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 523         req = mux.SetURLVars(req, vars)
 524         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 525         response = executeRequest(req, handleFunc)
 526         checkResponseCode(t, http.StatusOK, response.Code)
 527
 528         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 529         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 530
 531         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 532         defer ts.Close()
 533
 534         // Raise two alarms. The first should be delayed
 535         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 536         assert.Nil(t, alarmer.Raise(a), "raise failed")
 537         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 538
 539         b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 540         assert.Nil(t, alarmer.Raise(b), "raise failed")
 541         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 542
 543         // Clear two alarms. The first should be delayed. Check the alarms are removed
 544         assert.Nil(t, alarmer.Clear(a), "clear failed")
 545         assert.Nil(t, alarmer.Clear(b), "clear failed")
 546
 547         time.Sleep(time.Duration(2) * time.Second)
 548         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 549         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 550 }
 551
 552 func TestClearExpiredAlarms(t *testing.T) {
 553         xapp.Logger.Info("TestClearExpiredAlarms")
 554
 555         a := alarm.AlarmMessage{
 556                 Alarm:       alarmer.NewAlarm(72004, alarm.SeverityWarning, "threshold", ""),
 557                 AlarmAction: alarm.AlarmActionRaise,
 558                 AlarmTime:   time.Now().UnixNano(),
 559         }
 560         d := alarm.RICAlarmDefinitions[72004]
 561         n := AlarmNotification{a, *d}
 562         alarmManager.activeAlarms = make([]AlarmNotification, 0)
 563         alarmManager.UpdateActiveAlarmList(&n)
 564
 565         // Unknown SP
 566         a.Alarm.SpecificProblem = 1234
 567         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 568
 569         // TTL is 0
 570         d.TimeToLive = 0
 571         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 572
 573         // TTL not expired
 574         a.Alarm.SpecificProblem = 72004
 575         d.TimeToLive = 2
 576         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 577
 578         // TTL expired, alarm should be cleared
 579         time.Sleep(time.Duration(3) * time.Second)
 580         assert.Equal(t, len(alarmManager.activeAlarms), 1)
 581         assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 582         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 583 }
 584
 585 func TestSetAlarmConfig(t *testing.T) {
 586         xapp.Logger.Info("TestSetAlarmConfig")
 587
 588         var setAlarmConfig alarm.AlarmConfigParams
 589         setAlarmConfig.MaxActiveAlarms = 500
 590         setAlarmConfig.MaxAlarmHistory = 2000
 591
 592         pbodyEn, _ := json.Marshal(setAlarmConfig)
 593         req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 594         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 595         response := executeRequest(req, handleFunc)
 596         checkResponseCode(t, http.StatusOK, response.Code)
 597
 598         var getAlarmConfig alarm.AlarmConfigParams
 599         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 600         req = mux.SetURLVars(req, nil)
 601         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 602         response = executeRequest(req, handleFunc)
 603         checkResponseCode(t, http.StatusOK, response.Code)
 604
 605         // Decode the json output from handler
 606         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 607         if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 {
 608                 t.Errorf("Incorrect alarm thresholds")
 609         }
 610
 611         // Revert ot default
 612         setAlarmConfig.MaxActiveAlarms = 5000
 613         setAlarmConfig.MaxAlarmHistory = 20000
 614
 615         pbodyEn, _ = json.Marshal(setAlarmConfig)
 616         req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 617         handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig)
 618         response = executeRequest(req, handleFunc)
 619         checkResponseCode(t, http.StatusOK, response.Code)
 620
 621         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 622         req = mux.SetURLVars(req, nil)
 623         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 624         response = executeRequest(req, handleFunc)
 625         checkResponseCode(t, http.StatusOK, response.Code)
 626
 627         // Decode the json output from handler
 628         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 629         if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 {
 630                 t.Errorf("Incorrect alarm thresholds")
 631         }
 632 }
 633
 634 func TestConfigChangeCB(t *testing.T) {
 635         xapp.Logger.Info("TestConfigChangeCB")
 636         alarmManager.ConfigChangeCB("AlarmManager")
 637 }
 638
 639 func TestPostAlarm(t *testing.T) {
 640         xapp.Logger.Info("TestPostAlarm")
 641         var activeAlarms []AlarmNotification
 642         activeAlarms = make([]AlarmNotification, 1)
 643         alarmManager.PostAlarm(&activeAlarms[0])
 644 }
 645
 646 func TestPostAlarm1(t *testing.T) {
 647         xapp.Logger.Info("TestPostAlarm")
 648         var activeAlarms []AlarmNotification
 649         activeAlarms = make([]AlarmNotification, 2)
 650         alarmManager.PostAlarm(&activeAlarms[0])
 651 }
 652
 653 func TestNewAlarmManagerOther(t *testing.T){
 654     NewAlarmManager("", 0, true)
 655 }
 656
 657 func TestStatusCallbackFailure(t *testing.T) {
 658         xapp.Logger.Info("TestStatusCallbackFailure")
 659         alarmManager.rmrReady = false
 660         assert.Equal(t, false, alarmManager.StatusCB())
 661 }
 662
 663 func TestConfigChangeCBFailure(t *testing.T) {
 664         xapp.Logger.Info("TestConfigChangeCBFailure")
 665         alarmManager.maxActiveAlarms = 0
 666         alarmManager.maxAlarmHistory = 0
 667         alarmManager.ConfigChangeCB("AlarmManager")
 668 }
 669
 670 func TestReadAlarmDefinitionFromJsonWrongFilename(t *testing.T) {
 671   // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 672     xapp.Logger.Info("TestReadAlarmDefinitionFromJsonWrongFilename")
 673     os.Setenv("DEF_FILE","test.json")
 674     alarmManager.ReadAlarmDefinitionFromJson()
 675   // correct the filename
 676 }
 677
 678 func TestReadAlarmDefinitionFromJsonInvalidFilename(t *testing.T) {
 679   // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 680     xapp.Logger.Info("TestReadAlarmDefinitionFromJsonInvalidFilename")
 681     os.Setenv("DEF_FILE","../../definitions/test.json")
 682     alarmManager.ReadAlarmDefinitionFromJson()
 683   // correct the filename
 684 }
 685
 686 func TestPersistentStorage(t *testing.T) {
 687     xapp.Logger.Info("TestPersistentStorage")
 688     alarmManager.alarmInfoPvFile = "../../definitions/sample.json"
 689     alarmManager.ReadAlarmInfoFromPersistentVolume()
 690 }
 691
 692 func TestDeleteAlarmDefinitions1(t *testing.T) {
 693         xapp.Logger.Info("TestDeleteAlarmDefinitions1")
 694         //Get all
 695         //Delete Alarm which doesn't present
 696         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 697         vars := map[string]string{"alarmId": strconv.FormatUint(882004, 10)}
 698         req = mux.SetURLVars(req, vars)
 699         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 700         response := executeRequest(req, handleFunc)
 701         checkResponseCode(t, http.StatusOK, response.Code)
 702
 703         //Delete Alarm which is incorrect present
 704         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 705         vars = map[string]string{"alarmId": strconv.FormatUint(898989, 8)}
 706         req = mux.SetURLVars(req, vars)
 707         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 708         response = executeRequest(req, handleFunc)
 709         checkResponseCode(t, http.StatusOK, response.Code)
 710 }
 711
 712 func TestGetPreDefinedAlarmInvalidAlarm(t *testing.T) {
 713        xapp.Logger.Info("TestGetPreDefinedAlarmInvalidAlarm")
 714        req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 715        vars := map[string]string{"alarmId": "asdsc"}
 716        req = mux.SetURLVars(req, vars)
 717        handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 718        response := executeRequest(req, handleFunc)
 719        xapp.Logger.Info("response code = %v",response.Code)
 720        checkResponseCode(t, http.StatusBadRequest, response.Code)
 721 }
 722
 723 func TestDeleteAlarmDefinitions2(t *testing.T) {
 724        xapp.Logger.Info("TestDeleteAlarmDefinitions2")
 725        req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 726        //Giving Wrong alarmId which can't convert into int
 727        vars := map[string]string{"alarmId": "asdsc"}
 728        req = mux.SetURLVars(req, vars)
 729        handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 730        response := executeRequest(req, handleFunc)
 731        checkResponseCode(t, http.StatusBadRequest, response.Code)
 732 }
 733
 734 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 735         receivedAlert := waitForEvent()
 736
 737         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 738         _, ok := alarmManager.IsMatchFound(a)
 739         assert.True(t, ok)
 740
 741         return receivedAlert
 742 }
 743
 744 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 745         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 746         //assert.Equal(t, receivedAlert, e)
 747 }
 748
 749 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 750         l, err := net.Listen("tcp", "localhost:9093")
 751         if err != nil {
 752                 t.Error("Failed to create listener: " + err.Error())
 753         }
 754         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 755                 assert.Equal(t, r.Method, method)
 756                 assert.Equal(t, r.URL.String(), url)
 757
 758                 fireEvent(t, r.Body)
 759
 760                 w.Header().Add("Content-Type", "application/json")
 761                 w.WriteHeader(status)
 762                 b, _ := json.Marshal(respData)
 763                 w.Write(b)
 764         }))
 765         ts.Listener.Close()
 766         ts.Listener = l
 767
 768         ts.Start()
 769
 770         return ts
 771 }
 772
 773 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
 774         l, err := net.Listen("tcp", "localhost:9093")
 775         if err != nil {
 776                 t.Error("Failed to create listener: " + err.Error())
 777         }
 778         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 779                 assert.Equal(t, r.Method, method)
 780                 assert.Equal(t, r.URL.String(), url)
 781
 782                 w.Header().Add("Content-Type", "application/json")
 783                 w.WriteHeader(200)
 784                 // Read alerts from file
 785                 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
 786                 if err != nil {
 787                         t.Error("Failed to send response: ", err)
 788                 }
 789                 _, err = w.Write(payload)
 790                 if err != nil {
 791                         t.Error("Failed to send response: " + err.Error())
 792                 }
 793         }))
 794         ts.Listener.Close()
 795         ts.Listener = l
 796         ts.Start()
 797         return ts
 798 }
 799
 800 func waitForEvent() string {
 801         receivedAlert := <-eventChan
 802         return receivedAlert
 803 }
 804
 805 func fireEvent(t *testing.T, body io.ReadCloser) {
 806         reqBody, err := ioutil.ReadAll(body)
 807         assert.Nil(t, err, "ioutil.ReadAll failed")
 808         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 809
 810         eventChan <- fmt.Sprintf("%s", reqBody)
 811 }
 812
 813 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 814         rr := httptest.NewRecorder()
 815
 816         handleR.ServeHTTP(rr, req)
 817
 818         return rr
 819 }
 820
 821 func checkResponseCode(t *testing.T, expected, actual int) bool {
 822         if expected != actual {
 823                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 824                 return false
 825         }
 826         return true
 827 }
 828
 829 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
 830         go func() {
 831                 xapp.Logger.Info("Giving CLI command")
 832                 cmd := exec.Command(command, args...)
 833                 cmd.Dir = "../"
 834                 output, err := cmd.CombinedOutput()
 835                 if err != nil {
 836                         xapp.Logger.Info("CLI command failed out: %s", err)
 837                 }
 838                 xapp.Logger.Info("CLI command output: %s", output)
 839                 commandReady <- true
 840                 xapp.Logger.Info("CLI command completed")
 841         }()
 842 }
 843
 844 func readJSONFromFile(filename string) ([]byte, error) {
 845         file, err := ioutil.ReadFile(filename)
 846         if err != nil {
 847                 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
 848                 return nil, err
 849         }
 850         return file, nil
 851 }
 852
 853