manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io"
  28         "io/ioutil"
  29         "net"
  30         "net/http"
  31         "net/http/httptest"
  32         "os"
  33         "os/exec"
  34         "strconv"
  35         "strings"
  36         "testing"
  37         "time"
  38
  39         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go.git/alarm"
  40         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  41         "github.com/gorilla/mux"
  42         "github.com/prometheus/alertmanager/api/v2/models"
  43         "github.com/stretchr/testify/assert"
  44 )
  45
  46 var alarmManager *AlarmManager
  47 var alarmer *alarm.RICAlarm
  48 var eventChan chan string
  49
  50 // Test cases
  51 func TestMain(M *testing.M) {
  52         alarmManager = NewAlarmManager("localhost:9093", 500, false)
  53         alarmManager.alertInterval = 20000
  54         go alarmManager.Run(false, 5)
  55         time.Sleep(time.Duration(10) * time.Second)
  56
  57         // Wait until RMR is up-and-running
  58         for !xapp.Rmr.IsReady() {
  59                 time.Sleep(time.Duration(1) * time.Second)
  60         }
  61
  62         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  63         alarmManager.alarmClient = alarmer
  64         time.Sleep(time.Duration(5) * time.Second)
  65         eventChan = make(chan string)
  66
  67         os.Exit(M.Run())
  68 }
  69
  70 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  71         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  72         var alarmDefinition alarm.AlarmDefinition
  73         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  74         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
  75         req = mux.SetURLVars(req, vars)
  76         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  77         response := executeRequest(req, handleFunc)
  78         checkResponseCode(t, http.StatusOK, response.Code)
  79         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  80         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  81         if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" {
  82                 t.Errorf("Incorrect alarm definition")
  83         }
  84 }
  85
  86 func TestSetAlarmDefinitions(t *testing.T) {
  87         xapp.Logger.Info("TestSetAlarmDefinitions")
  88
  89         var alarm72004Definition alarm.AlarmDefinition
  90         alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM
  91         alarm72004Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB/G-NODEB"
  92         alarm72004Definition.EventType = "Communication error"
  93         alarm72004Definition.OperationInstructions = "Not defined"
  94         alarm72004Definition.RaiseDelay = 0
  95         alarm72004Definition.ClearDelay = 0
  96
  97         var alarm72008Definition alarm.AlarmDefinition
  98         alarm72008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
  99         alarm72008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 100         alarm72008Definition.EventType = "storage warning"
 101         alarm72008Definition.OperationInstructions = "Clear alarms or raise threshold"
 102         alarm72008Definition.RaiseDelay = 0
 103         alarm72008Definition.ClearDelay = 0
 104
 105         var alarm72009Definition alarm.AlarmDefinition
 106         alarm72009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 107         alarm72009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 108         alarm72009Definition.EventType = "storage warning"
 109         alarm72009Definition.OperationInstructions = "Clear alarms or raise threshold"
 110         alarm72009Definition.RaiseDelay = 0
 111         alarm72009Definition.ClearDelay = 0
 112
 113         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition, &alarm72008Definition, &alarm72009Definition}}
 114         pbodyEn, _ := json.Marshal(pbodyParams)
 115         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 116         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 117         response := executeRequest(req, handleFunc)
 118         status := checkResponseCode(t, http.StatusOK, response.Code)
 119         xapp.Logger.Info("status = %v", status)
 120 }
 121
 122 func TestSetAlarmConfigDecodeError(t *testing.T) {
 123         xapp.Logger.Info("TestSetAlarmConfigDecodeError")
 124
 125         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 126         req, _ := http.NewRequest("POST", "/ric/v1/alarms", bytes.NewBuffer(jsonStr))
 127         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 128         response := executeRequest(req, handleFunc)
 129         status := checkResponseCode(t, http.StatusOK, response.Code)
 130         xapp.Logger.Info("status = %v", status)
 131 }
 132
 133 func TestSetAlarmDefinitionDecodeError(t *testing.T) {
 134         xapp.Logger.Info("TestSetAlarmDefinitionDecodeError")
 135
 136         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 137         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(jsonStr))
 138         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 139         response := executeRequest(req, handleFunc)
 140         status := checkResponseCode(t, http.StatusBadRequest, response.Code)
 141         xapp.Logger.Info("status = %v", status)
 142 }
 143
 144 func TestRaiseAlarmEmptyBody(t *testing.T) {
 145         xapp.Logger.Info("TestRaiseAlarmEmptyBody")
 146         req, _ := http.NewRequest("POST", "/ric/v1/alarms", nil)
 147         handleFunc := http.HandlerFunc(alarmManager.RaiseAlarm)
 148         response := executeRequest(req, handleFunc)
 149         status := checkResponseCode(t, http.StatusOK, response.Code)
 150         xapp.Logger.Info("status = %v", status)
 151 }
 152
 153 func TestSetAlarmDefinitionsEmptyBody(t *testing.T) {
 154         xapp.Logger.Info("TestSetAlarmDefinitionsEmptyBody")
 155         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", nil)
 156         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 157         response := executeRequest(req, handleFunc)
 158         status := checkResponseCode(t, http.StatusBadRequest, response.Code)
 159         xapp.Logger.Info("status = %v", status)
 160 }
 161
 162 func TestClearAlarmEmptyBody(t *testing.T) {
 163         xapp.Logger.Info("TestClearAlarmEmptyBody")
 164         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms", nil)
 165         handleFunc := http.HandlerFunc(alarmManager.ClearAlarm)
 166         response := executeRequest(req, handleFunc)
 167         status := checkResponseCode(t, http.StatusOK, response.Code)
 168         xapp.Logger.Info("status = %v", status)
 169 }
 170
 171 func TestGetAlarmDefinitions(t *testing.T) {
 172         xapp.Logger.Info("TestGetAlarmDefinitions")
 173         var alarmDefinition alarm.AlarmDefinition
 174         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 175         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 176         req = mux.SetURLVars(req, vars)
 177         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 178         response := executeRequest(req, handleFunc)
 179         checkResponseCode(t, http.StatusOK, response.Code)
 180         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 181         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 182         if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" {
 183                 t.Errorf("Incorrect alarm definition")
 184         }
 185 }
 186
 187 func TestDeleteAlarmDefinitions(t *testing.T) {
 188         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 189         //Get all
 190         var ricAlarmDefinitions RicAlarmDefinitions
 191         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 192         req = mux.SetURLVars(req, nil)
 193         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 194         response := executeRequest(req, handleFunc)
 195         checkResponseCode(t, http.StatusOK, response.Code)
 196         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 197         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 198                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 199         }
 200
 201         //Delete 72004
 202         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 203         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 204         req = mux.SetURLVars(req, vars)
 205         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 206         response = executeRequest(req, handleFunc)
 207         checkResponseCode(t, http.StatusOK, response.Code)
 208
 209         //Get 72004 fail
 210         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 211         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 212         req = mux.SetURLVars(req, vars)
 213         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 214         response = executeRequest(req, handleFunc)
 215         checkResponseCode(t, http.StatusBadRequest, response.Code)
 216
 217         //Delete Alarm which doesn't present
 218         //Set 72004 success
 219         var alarm72004Definition alarm.AlarmDefinition
 220         alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM
 221         alarm72004Definition.AlarmText = "E2 CONNECTION PROBLEM"
 222         alarm72004Definition.EventType = "Processing error"
 223         alarm72004Definition.OperationInstructions = "Not defined"
 224         alarm72004Definition.RaiseDelay = 0
 225         alarm72004Definition.ClearDelay = 0
 226         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition}}
 227         pbodyEn, _ := json.Marshal(pbodyParams)
 228         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 229         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 230         response = executeRequest(req, handleFunc)
 231         checkResponseCode(t, http.StatusOK, response.Code)
 232
 233         //Get 72004 success
 234         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 235         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 236         req = mux.SetURLVars(req, vars)
 237         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 238         response = executeRequest(req, handleFunc)
 239         checkResponseCode(t, http.StatusOK, response.Code)
 240 }
 241
 242 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 243         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 244         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 245         defer ts.Close()
 246
 247         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 248         assert.Nil(t, alarmer.Raise(a), "raise failed")
 249
 250         VerifyAlarm(t, a, 1)
 251
 252         var activeAlarms []AlarmNotification
 253         activeAlarms = make([]AlarmNotification, 1)
 254         req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil)
 255         req = mux.SetURLVars(req, nil)
 256         handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms)
 257         response := executeRequest(req, handleFunc)
 258         checkResponseCode(t, http.StatusOK, response.Code)
 259
 260         // Decode the json output from handler
 261         json.NewDecoder(response.Body).Decode(activeAlarms)
 262         if len(activeAlarms) != 1 {
 263                 t.Errorf("Incorrect alarm alarm count")
 264         }
 265
 266         var alarmHistory []AlarmNotification
 267         alarmHistory = make([]AlarmNotification, 1)
 268         req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil)
 269         req = mux.SetURLVars(req, nil)
 270         handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory)
 271         response = executeRequest(req, handleFunc)
 272         checkResponseCode(t, http.StatusOK, response.Code)
 273
 274         // Decode the json output from handler
 275         json.NewDecoder(response.Body).Decode(alarmHistory)
 276         if len(alarmHistory) != 1 {
 277                 t.Errorf("Incorrect alarm history count")
 278         }
 279 }
 280
 281 func TestAlarmClearedSucess(t *testing.T) {
 282         xapp.Logger.Info("TestAlarmClearedSucess")
 283         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 284         defer ts.Close()
 285
 286         // Raise the alarm
 287         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 288         assert.Nil(t, alarmer.Raise(a), "raise failed")
 289
 290         VerifyAlarm(t, a, 1)
 291
 292         // Now Clear the alarm and check alarm is removed
 293         a = alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 294         assert.Nil(t, alarmer.Clear(a), "clear failed")
 295
 296         time.Sleep(time.Duration(2) * time.Second)
 297         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 298 }
 299
 300 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 301         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 302         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 303         defer ts.Close()
 304
 305         // Raise two alarms
 306         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 307         assert.Nil(t, alarmer.Raise(a), "raise failed")
 308
 309         b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11")
 310         assert.Nil(t, alarmer.Raise(b), "raise failed")
 311
 312         time.Sleep(time.Duration(5) * time.Second)
 313
 314         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 315         VerifyAlarm(t, a, 1)
 316         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 317         VerifyAlarm(t, b, 2)
 318 }
 319
 320 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 321         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 322         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 323         defer ts.Close()
 324
 325         // Raise two alarms
 326         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 327         assert.Nil(t, alarmer.Clear(a), "clear failed")
 328
 329         b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11")
 330         assert.Nil(t, alarmer.Clear(b), "clear failed")
 331
 332         time.Sleep(time.Duration(2) * time.Second)
 333         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 334 }
 335
 336 func TestAlarmsSuppresedSucess(t *testing.T) {
 337         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 338         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 339         defer ts.Close()
 340
 341         // Raise two similar/matching alarms ... the second one suppresed
 342         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 343         assert.Nil(t, alarmer.Raise(a), "raise failed")
 344         assert.Nil(t, alarmer.Raise(a), "raise failed")
 345
 346         VerifyAlarm(t, a, 1)
 347         assert.Nil(t, alarmer.Clear(a), "clear failed")
 348 }
 349
 350 func TestInvalidAlarms(t *testing.T) {
 351         xapp.Logger.Info("TestInvalidAlarms")
 352         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 353         assert.Nil(t, alarmer.Raise(a), "raise failed")
 354         time.Sleep(time.Duration(2) * time.Second)
 355 }
 356
 357 func TestAlarmHandlingErrorCases(t *testing.T) {
 358         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 359         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 360         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 361         assert.Nil(t, ok, "raise failed")
 362 }
 363
 364 func TestConsumeUnknownMessage(t *testing.T) {
 365         xapp.Logger.Info("TestConsumeUnknownMessage")
 366         err := alarmManager.Consume(&xapp.RMRParams{})
 367         assert.Nil(t, err, "raise failed")
 368 }
 369
 370 func TestStatusCallback(t *testing.T) {
 371         xapp.Logger.Info("TestStatusCallback")
 372         assert.Equal(t, true, alarmManager.StatusCB())
 373 }
 374
 375 func TestActiveAlarmMaxThresholds(t *testing.T) {
 376         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 377         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 378         alarmManager.maxActiveAlarms = 0
 379         alarmManager.maxAlarmHistory = 10
 380
 381         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 382         assert.Nil(t, alarmer.Raise(a), "raise failed")
 383
 384         var alarmConfigParams alarm.AlarmConfigParams
 385         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 386         req = mux.SetURLVars(req, nil)
 387         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 388         response := executeRequest(req, handleFunc)
 389
 390         // Check HTTP Status Code
 391         checkResponseCode(t, http.StatusOK, response.Code)
 392
 393         // Decode the json output from handler
 394         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 395         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 396                 t.Errorf("Incorrect alarm thresholds")
 397         }
 398
 399         time.Sleep(time.Duration(1) * time.Second)
 400         alarmManager.maxActiveAlarms = 5000
 401         alarmManager.maxAlarmHistory = 20000
 402         VerifyAlarm(t, a, 2)
 403         VerifyAlarm(t, a, 2)
 404         ts.Close()
 405 }
 406
 407 func TestGetPrometheusAlerts(t *testing.T) {
 408         time.Sleep(1 * time.Second)
 409         xapp.Logger.Info("TestGetPrometheusAlerts")
 410         ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
 411
 412         commandReady := make(chan bool, 1)
 413         command := "cli/alarm-cli"
 414         args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
 415         ExecCLICommand(commandReady, command, args...)
 416         <-commandReady
 417
 418         ts.Close()
 419 }
 420
 421 func TestDelayedAlarmRaiseAndClear(t *testing.T) {
 422         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
 423
 424         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 425         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 426
 427         // Add new alarm definition
 428         var alarm9999Definition alarm.AlarmDefinition
 429         alarm9999Definition.AlarmId = 9999
 430         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 431         alarm9999Definition.EventType = "Test type"
 432         alarm9999Definition.OperationInstructions = "Not defined"
 433         alarm9999Definition.RaiseDelay = 1
 434         alarm9999Definition.ClearDelay = 1
 435         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 436         pbodyEn, _ := json.Marshal(pbodyParams)
 437         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 438         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 439         response := executeRequest(req, handleFunc)
 440         checkResponseCode(t, http.StatusOK, response.Code)
 441
 442         // Verify 9999 alarm definition
 443         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 444         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 445         req = mux.SetURLVars(req, vars)
 446         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 447         response = executeRequest(req, handleFunc)
 448         checkResponseCode(t, http.StatusOK, response.Code)
 449
 450         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 451         defer ts.Close()
 452
 453         // Raise alarm. Posting alert and updating alarm history should be delayed
 454         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 455         assert.Nil(t, alarmer.Raise(a), "raise failed")
 456         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 457
 458         // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
 459         assert.Nil(t, alarmer.Clear(a), "clear failed")
 460
 461         time.Sleep(time.Duration(2) * time.Second)
 462         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 463         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
 464 }
 465
 466 func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
 467         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
 468
 469         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 470         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 471
 472         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 473         defer ts.Close()
 474
 475         // Raise two alarms. The first should be delayed
 476         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 477         assert.Nil(t, alarmer.Raise(a), "raise failed")
 478         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 479
 480         b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 481         assert.Nil(t, alarmer.Raise(b), "raise failed")
 482         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 483
 484         // Clear two alarms. The first should be delayed. Check the alarms are removed
 485         assert.Nil(t, alarmer.Clear(a), "clear failed")
 486         assert.Nil(t, alarmer.Clear(b), "clear failed")
 487
 488         time.Sleep(time.Duration(2) * time.Second)
 489         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 490         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 491 }
 492
 493 func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
 494         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
 495
 496         // Delete exisitng 9999 alarm definition
 497         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 498         vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
 499         req = mux.SetURLVars(req, vars)
 500         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 501         response := executeRequest(req, handleFunc)
 502         checkResponseCode(t, http.StatusOK, response.Code)
 503
 504         // Add updated 9999 alarm definition
 505         var alarm9999Definition alarm.AlarmDefinition
 506         alarm9999Definition.AlarmId = 9999
 507         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 508         alarm9999Definition.EventType = "Test type"
 509         alarm9999Definition.OperationInstructions = "Not defined"
 510         alarm9999Definition.RaiseDelay = 1
 511         alarm9999Definition.ClearDelay = 0
 512         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 513         pbodyEn, _ := json.Marshal(pbodyParams)
 514         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 515         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 516         response = executeRequest(req, handleFunc)
 517         checkResponseCode(t, http.StatusOK, response.Code)
 518
 519         // Verify 9999 alarm definition
 520         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 521         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 522         req = mux.SetURLVars(req, vars)
 523         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 524         response = executeRequest(req, handleFunc)
 525         checkResponseCode(t, http.StatusOK, response.Code)
 526
 527         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 528         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 529
 530         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 531         defer ts.Close()
 532
 533         // Raise two alarms. The first should be delayed
 534         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 535         assert.Nil(t, alarmer.Raise(a), "raise failed")
 536         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 537
 538         b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 539         assert.Nil(t, alarmer.Raise(b), "raise failed")
 540         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 541
 542         // Clear two alarms. The first should be delayed. Check the alarms are removed
 543         assert.Nil(t, alarmer.Clear(a), "clear failed")
 544         assert.Nil(t, alarmer.Clear(b), "clear failed")
 545
 546         time.Sleep(time.Duration(2) * time.Second)
 547         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 548         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 549 }
 550
 551 func TestClearExpiredAlarms(t *testing.T) {
 552         xapp.Logger.Info("TestClearExpiredAlarms")
 553
 554         a := alarm.AlarmMessage{
 555                 Alarm:       alarmer.NewAlarm(72004, alarm.SeverityWarning, "threshold", ""),
 556                 AlarmAction: alarm.AlarmActionRaise,
 557                 AlarmTime:   time.Now().UnixNano(),
 558         }
 559         d := alarm.RICAlarmDefinitions[72004]
 560         n := AlarmNotification{a, *d}
 561         alarmManager.activeAlarms = make([]AlarmNotification, 0)
 562         alarmManager.UpdateActiveAlarmList(&n)
 563
 564         // Unknown SP
 565         a.Alarm.SpecificProblem = 1234
 566         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 567
 568         // TTL is 0
 569         d.TimeToLive = 0
 570         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 571
 572         // TTL not expired
 573         a.Alarm.SpecificProblem = 72004
 574         d.TimeToLive = 2
 575         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 576
 577         // TTL expired, alarm should be cleared
 578         time.Sleep(time.Duration(3) * time.Second)
 579         assert.Equal(t, len(alarmManager.activeAlarms), 1)
 580         assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 581         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 582 }
 583
 584 func TestSetAlarmConfig(t *testing.T) {
 585         xapp.Logger.Info("TestSetAlarmConfig")
 586
 587         var setAlarmConfig alarm.AlarmConfigParams
 588         setAlarmConfig.MaxActiveAlarms = 500
 589         setAlarmConfig.MaxAlarmHistory = 2000
 590
 591         pbodyEn, _ := json.Marshal(setAlarmConfig)
 592         req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 593         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 594         response := executeRequest(req, handleFunc)
 595         checkResponseCode(t, http.StatusOK, response.Code)
 596
 597         var getAlarmConfig alarm.AlarmConfigParams
 598         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 599         req = mux.SetURLVars(req, nil)
 600         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 601         response = executeRequest(req, handleFunc)
 602         checkResponseCode(t, http.StatusOK, response.Code)
 603
 604         // Decode the json output from handler
 605         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 606         if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 {
 607                 t.Errorf("Incorrect alarm thresholds")
 608         }
 609
 610         // Revert ot default
 611         setAlarmConfig.MaxActiveAlarms = 5000
 612         setAlarmConfig.MaxAlarmHistory = 20000
 613
 614         pbodyEn, _ = json.Marshal(setAlarmConfig)
 615         req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 616         handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig)
 617         response = executeRequest(req, handleFunc)
 618         checkResponseCode(t, http.StatusOK, response.Code)
 619
 620         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 621         req = mux.SetURLVars(req, nil)
 622         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 623         response = executeRequest(req, handleFunc)
 624         checkResponseCode(t, http.StatusOK, response.Code)
 625
 626         // Decode the json output from handler
 627         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 628         if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 {
 629                 t.Errorf("Incorrect alarm thresholds")
 630         }
 631 }
 632
 633 func TestConfigChangeCB(t *testing.T) {
 634         xapp.Logger.Info("TestConfigChangeCB")
 635         alarmManager.ConfigChangeCB("AlarmManager")
 636 }
 637
 638 func TestPostAlarm(t *testing.T) {
 639         xapp.Logger.Info("TestPostAlarm")
 640         var activeAlarms []AlarmNotification
 641         activeAlarms = make([]AlarmNotification, 1)
 642         alarmManager.PostAlarm(&activeAlarms[0])
 643 }
 644
 645 func TestPostAlarm1(t *testing.T) {
 646         xapp.Logger.Info("TestPostAlarm")
 647         var activeAlarms []AlarmNotification
 648         activeAlarms = make([]AlarmNotification, 2)
 649         alarmManager.PostAlarm(&activeAlarms[0])
 650 }
 651
 652 func TestNewAlarmManagerOther(t *testing.T) {
 653         NewAlarmManager("", 0, true)
 654 }
 655
 656 func TestStatusCallbackFailure(t *testing.T) {
 657         xapp.Logger.Info("TestStatusCallbackFailure")
 658         alarmManager.rmrReady = false
 659         assert.Equal(t, false, alarmManager.StatusCB())
 660 }
 661
 662 func TestConfigChangeCBFailure(t *testing.T) {
 663         xapp.Logger.Info("TestConfigChangeCBFailure")
 664         alarmManager.maxActiveAlarms = 0
 665         alarmManager.maxAlarmHistory = 0
 666         alarmManager.ConfigChangeCB("AlarmManager")
 667 }
 668
 669 func TestReadAlarmDefinitionFromJsonWrongFilename(t *testing.T) {
 670         // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 671         xapp.Logger.Info("TestReadAlarmDefinitionFromJsonWrongFilename")
 672         os.Setenv("DEF_FILE", "test.json")
 673         alarmManager.ReadAlarmDefinitionFromJson()
 674         // correct the filename
 675 }
 676
 677 func TestReadAlarmDefinitionFromJsonInvalidFilename(t *testing.T) {
 678         // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 679         xapp.Logger.Info("TestReadAlarmDefinitionFromJsonInvalidFilename")
 680         os.Setenv("DEF_FILE", "../../definitions/test.json")
 681         alarmManager.ReadAlarmDefinitionFromJson()
 682         // correct the filename
 683 }
 684
 685 func TestPersistentStorage(t *testing.T) {
 686         xapp.Logger.Info("TestPersistentStorage")
 687         alarmManager.alarmInfoPvFile = "../../definitions/sample.json"
 688         alarmManager.ReadAlarmInfoFromPersistentVolume()
 689 }
 690
 691 func TestDeleteAlarmDefinitions1(t *testing.T) {
 692         xapp.Logger.Info("TestDeleteAlarmDefinitions1")
 693         //Get all
 694         //Delete Alarm which doesn't present
 695         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 696         vars := map[string]string{"alarmId": strconv.FormatUint(882004, 10)}
 697         req = mux.SetURLVars(req, vars)
 698         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 699         response := executeRequest(req, handleFunc)
 700         checkResponseCode(t, http.StatusOK, response.Code)
 701
 702         //Delete Alarm which is incorrect present
 703         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 704         vars = map[string]string{"alarmId": strconv.FormatUint(898989, 8)}
 705         req = mux.SetURLVars(req, vars)
 706         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 707         response = executeRequest(req, handleFunc)
 708         checkResponseCode(t, http.StatusOK, response.Code)
 709 }
 710
 711 func TestGetPreDefinedAlarmInvalidAlarm(t *testing.T) {
 712         xapp.Logger.Info("TestGetPreDefinedAlarmInvalidAlarm")
 713         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 714         vars := map[string]string{"alarmId": "asdsc"}
 715         req = mux.SetURLVars(req, vars)
 716         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 717         response := executeRequest(req, handleFunc)
 718         xapp.Logger.Info("response code = %v", response.Code)
 719         checkResponseCode(t, http.StatusBadRequest, response.Code)
 720 }
 721
 722 func TestDeleteAlarmDefinitions2(t *testing.T) {
 723         xapp.Logger.Info("TestDeleteAlarmDefinitions2")
 724         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 725         //Giving Wrong alarmId which can't convert into int
 726         vars := map[string]string{"alarmId": "asdsc"}
 727         req = mux.SetURLVars(req, vars)
 728         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 729         response := executeRequest(req, handleFunc)
 730         checkResponseCode(t, http.StatusBadRequest, response.Code)
 731 }
 732
 733 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 734         receivedAlert := waitForEvent()
 735
 736         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 737         _, ok := alarmManager.IsMatchFound(a)
 738         assert.True(t, ok)
 739
 740         return receivedAlert
 741 }
 742
 743 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 744         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 745         //assert.Equal(t, receivedAlert, e)
 746 }
 747
 748 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 749         l, err := net.Listen("tcp", "localhost:9093")
 750         if err != nil {
 751                 t.Error("Failed to create listener: " + err.Error())
 752         }
 753         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 754                 assert.Equal(t, r.Method, method)
 755                 assert.Equal(t, r.URL.String(), url)
 756
 757                 fireEvent(t, r.Body)
 758
 759                 w.Header().Add("Content-Type", "application/json")
 760                 w.WriteHeader(status)
 761                 b, _ := json.Marshal(respData)
 762                 w.Write(b)
 763         }))
 764         ts.Listener.Close()
 765         ts.Listener = l
 766
 767         ts.Start()
 768
 769         return ts
 770 }
 771
 772 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
 773         l, err := net.Listen("tcp", "localhost:9093")
 774         if err != nil {
 775                 t.Error("Failed to create listener: " + err.Error())
 776         }
 777         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 778                 assert.Equal(t, r.Method, method)
 779                 assert.Equal(t, r.URL.String(), url)
 780
 781                 w.Header().Add("Content-Type", "application/json")
 782                 w.WriteHeader(200)
 783                 // Read alerts from file
 784                 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
 785                 if err != nil {
 786                         t.Error("Failed to send response: ", err)
 787                 }
 788                 _, err = w.Write(payload)
 789                 if err != nil {
 790                         t.Error("Failed to send response: " + err.Error())
 791                 }
 792         }))
 793         ts.Listener.Close()
 794         ts.Listener = l
 795         ts.Start()
 796         return ts
 797 }
 798
 799 func waitForEvent() string {
 800         receivedAlert := <-eventChan
 801         return receivedAlert
 802 }
 803
 804 func fireEvent(t *testing.T, body io.ReadCloser) {
 805         reqBody, err := ioutil.ReadAll(body)
 806         assert.Nil(t, err, "ioutil.ReadAll failed")
 807         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 808
 809         eventChan <- fmt.Sprintf("%s", reqBody)
 810 }
 811
 812 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 813         rr := httptest.NewRecorder()
 814
 815         handleR.ServeHTTP(rr, req)
 816
 817         return rr
 818 }
 819
 820 func checkResponseCode(t *testing.T, expected, actual int) bool {
 821         if expected != actual {
 822                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 823                 return false
 824         }
 825         return true
 826 }
 827
 828 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
 829         go func() {
 830                 xapp.Logger.Info("Giving CLI command")
 831                 cmd := exec.Command(command, args...)
 832                 cmd.Dir = "../"
 833                 output, err := cmd.CombinedOutput()
 834                 if err != nil {
 835                         xapp.Logger.Info("CLI command failed out: %s", err)
 836                 }
 837                 xapp.Logger.Info("CLI command output: %s", output)
 838                 commandReady <- true
 839                 xapp.Logger.Info("CLI command completed")
 840         }()
 841 }
 842
 843 func readJSONFromFile(filename string) ([]byte, error) {
 844         file, err := ioutil.ReadFile(filename)
 845         if err != nil {
 846                 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
 847                 return nil, err
 848         }
 849         return file, nil
 850 }