manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io"
  28         "io/ioutil"
  29         "net"
  30         "net/http"
  31         "net/http/httptest"
  32         "os"
  33         "os/exec"
  34         "strconv"
  35         "strings"
  36         "testing"
  37         "time"
  38
  39         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  40         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  41         "github.com/gorilla/mux"
  42         "github.com/prometheus/alertmanager/api/v2/models"
  43         "github.com/stretchr/testify/assert"
  44 )
  45
  46 var alarmManager *AlarmManager
  47 var alarmer *alarm.RICAlarm
  48 var eventChan chan string
  49
  50 // Test cases
  51 func TestMain(M *testing.M) {
  52         alarmManager = NewAlarmManager("localhost:9093", 500, false)
  53         alarmManager.alertInterval = 20000
  54         go alarmManager.Run(false, 5)
  55         time.Sleep(time.Duration(10) * time.Second)
  56
  57         // Wait until RMR is up-and-running
  58         for !xapp.Rmr.IsReady() {
  59                 time.Sleep(time.Duration(1) * time.Second)
  60         }
  61
  62         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  63         alarmManager.alarmClient = alarmer
  64         time.Sleep(time.Duration(5) * time.Second)
  65         eventChan = make(chan string)
  66
  67         os.Exit(M.Run())
  68 }
  69
  70 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  71         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  72         var alarmDefinition alarm.AlarmDefinition
  73         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  74         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
  75         req = mux.SetURLVars(req, vars)
  76         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  77         response := executeRequest(req, handleFunc)
  78         checkResponseCode(t, http.StatusOK, response.Code)
  79         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  80         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  81     if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" {
  82                 t.Errorf("Incorrect alarm definition")
  83         }
  84 }
  85
  86 func TestSetAlarmDefinitions(t *testing.T) {
  87         xapp.Logger.Info("TestSetAlarmDefinitions")
  88
  89     var alarm72004Definition alarm.AlarmDefinition
  90         alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM
  91         alarm72004Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB/G-NODEB"
  92         alarm72004Definition.EventType = "Communication error"
  93         alarm72004Definition.OperationInstructions = "Not defined"
  94         alarm72004Definition.RaiseDelay = 0
  95         alarm72004Definition.ClearDelay = 0
  96
  97         var alarm72008Definition alarm.AlarmDefinition
  98         alarm72008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
  99         alarm72008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 100         alarm72008Definition.EventType = "storage warning"
 101         alarm72008Definition.OperationInstructions = "Clear alarms or raise threshold"
 102         alarm72008Definition.RaiseDelay = 0
 103         alarm72008Definition.ClearDelay = 0
 104
 105         var alarm72009Definition alarm.AlarmDefinition
 106         alarm72009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 107         alarm72009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 108         alarm72009Definition.EventType = "storage warning"
 109         alarm72009Definition.OperationInstructions = "Clear alarms or raise threshold"
 110         alarm72009Definition.RaiseDelay = 0
 111         alarm72009Definition.ClearDelay = 0
 112
 113         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition, &alarm72008Definition, &alarm72009Definition}}
 114         pbodyEn, _ := json.Marshal(pbodyParams)
 115         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 116         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 117         response := executeRequest(req, handleFunc)
 118         status := checkResponseCode(t, http.StatusOK, response.Code)
 119         xapp.Logger.Info("status = %v", status)
 120 }
 121
 122
 123 func TestSetAlarmConfigDecodeError(t *testing.T) {
 124         xapp.Logger.Info("TestSetAlarmConfigDecodeError")
 125
 126         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 127         req, _ := http.NewRequest("POST", "/ric/v1/alarms", bytes.NewBuffer(jsonStr))
 128         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 129         response := executeRequest(req, handleFunc)
 130         status := checkResponseCode(t,http.StatusOK,response.Code)
 131         xapp.Logger.Info("status = %v", status)
 132 }
 133
 134 func TestSetAlarmDefinitionDecodeError(t *testing.T) {
 135         xapp.Logger.Info("TestSetAlarmDefinitionDecodeError")
 136
 137         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 138         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(jsonStr))
 139         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 140         response := executeRequest(req, handleFunc)
 141         status := checkResponseCode(t,http.StatusBadRequest,response.Code)
 142         xapp.Logger.Info("status = %v", status)
 143 }
 144
 145 func TestRaiseAlarmEmptyBody(t *testing.T) {
 146         xapp.Logger.Info("TestRaiseAlarmEmptyBody")
 147         req, _ := http.NewRequest("POST", "/ric/v1/alarms", nil)
 148         handleFunc := http.HandlerFunc(alarmManager.RaiseAlarm)
 149         response := executeRequest(req, handleFunc)
 150         status := checkResponseCode(t,http.StatusOK,response.Code)
 151         xapp.Logger.Info("status = %v", status)
 152 }
 153
 154 func TestSetAlarmDefinitionsEmptyBody(t *testing.T) {
 155         xapp.Logger.Info("TestSetAlarmDefinitionsEmptyBody")
 156         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", nil)
 157         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 158         response := executeRequest(req, handleFunc)
 159         status := checkResponseCode(t,http.StatusBadRequest,response.Code)
 160         xapp.Logger.Info("status = %v", status)
 161 }
 162
 163 func TestClearAlarmEmptyBody(t *testing.T) {
 164         xapp.Logger.Info("TestClearAlarmEmptyBody")
 165         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms", nil)
 166         handleFunc := http.HandlerFunc(alarmManager.ClearAlarm)
 167         response := executeRequest(req, handleFunc)
 168         status := checkResponseCode(t,http.StatusOK,response.Code)
 169         xapp.Logger.Info("status = %v", status)
 170 }
 171
 172 func TestGetAlarmDefinitions(t *testing.T) {
 173         xapp.Logger.Info("TestGetAlarmDefinitions")
 174         var alarmDefinition alarm.AlarmDefinition
 175         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 176         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 177         req = mux.SetURLVars(req, vars)
 178         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 179         response := executeRequest(req, handleFunc)
 180         checkResponseCode(t, http.StatusOK, response.Code)
 181         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 182         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 183         if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" {
 184                 t.Errorf("Incorrect alarm definition")
 185         }
 186 }
 187
 188 func TestDeleteAlarmDefinitions(t *testing.T) {
 189         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 190         //Get all
 191         var ricAlarmDefinitions RicAlarmDefinitions
 192         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 193         req = mux.SetURLVars(req, nil)
 194         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 195         response := executeRequest(req, handleFunc)
 196         checkResponseCode(t, http.StatusOK, response.Code)
 197         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 198         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 199                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 200         }
 201
 202         //Delete 72004
 203         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 204         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 205         req = mux.SetURLVars(req, vars)
 206         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 207         response = executeRequest(req, handleFunc)
 208         checkResponseCode(t, http.StatusOK, response.Code)
 209
 210         //Get 72004 fail
 211         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 212         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 213         req = mux.SetURLVars(req, vars)
 214         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 215         response = executeRequest(req, handleFunc)
 216         checkResponseCode(t, http.StatusBadRequest, response.Code)
 217
 218         //Delete Alarm which doesn't present
 219         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 220         vars = map[string]string{"alarmId": strconv.FormatUint(882004, 10)}
 221         req = mux.SetURLVars(req, vars)
 222         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 223         response = executeRequest(req, handleFunc)
 224         checkResponseCode(t, http.StatusOK, response.Code)
 225
 226         //Delete Alarm which is incorrect present
 227         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 228         vars = map[string]string{"alarmId": strconv.FormatUint(1234, 8)}
 229         req = mux.SetURLVars(req, vars)
 230         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 231         response = executeRequest(req, handleFunc)
 232         checkResponseCode(t, http.StatusOK, response.Code)
 233
 234         //Set 72004 success
 235         var alarm72004Definition alarm.AlarmDefinition
 236         alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM
 237         alarm72004Definition.AlarmText = "E2 CONNECTION PROBLEM"
 238         alarm72004Definition.EventType = "Processing error"
 239         alarm72004Definition.OperationInstructions = "Not defined"
 240         alarm72004Definition.RaiseDelay = 0
 241         alarm72004Definition.ClearDelay = 0
 242         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition}}
 243         pbodyEn, _ := json.Marshal(pbodyParams)
 244         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 245         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 246         response = executeRequest(req, handleFunc)
 247         checkResponseCode(t, http.StatusOK, response.Code)
 248
 249         //Get 72004 success
 250         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 251         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 252         req = mux.SetURLVars(req, vars)
 253         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 254         response = executeRequest(req, handleFunc)
 255         checkResponseCode(t, http.StatusOK, response.Code)
 256 }
 257
 258 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 259         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 260         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 261         defer ts.Close()
 262
 263         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 264         assert.Nil(t, alarmer.Raise(a), "raise failed")
 265
 266         VerifyAlarm(t, a, 1)
 267
 268         var activeAlarms []AlarmNotification
 269         activeAlarms = make([]AlarmNotification, 1)
 270         req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil)
 271         req = mux.SetURLVars(req, nil)
 272         handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms)
 273         response := executeRequest(req, handleFunc)
 274         checkResponseCode(t, http.StatusOK, response.Code)
 275
 276         // Decode the json output from handler
 277         json.NewDecoder(response.Body).Decode(activeAlarms)
 278         if len(activeAlarms) != 1 {
 279                 t.Errorf("Incorrect alarm alarm count")
 280         }
 281
 282         var alarmHistory []AlarmNotification
 283         alarmHistory = make([]AlarmNotification, 1)
 284         req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil)
 285         req = mux.SetURLVars(req, nil)
 286         handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory)
 287         response = executeRequest(req, handleFunc)
 288         checkResponseCode(t, http.StatusOK, response.Code)
 289
 290         // Decode the json output from handler
 291         json.NewDecoder(response.Body).Decode(alarmHistory)
 292         if len(alarmHistory) != 1 {
 293                 t.Errorf("Incorrect alarm history count")
 294         }
 295 }
 296
 297 func TestAlarmClearedSucess(t *testing.T) {
 298         xapp.Logger.Info("TestAlarmClearedSucess")
 299         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 300         defer ts.Close()
 301
 302         // Raise the alarm
 303         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 304         assert.Nil(t, alarmer.Raise(a), "raise failed")
 305
 306         VerifyAlarm(t, a, 1)
 307
 308         // Now Clear the alarm and check alarm is removed
 309         a = alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1")
 310     assert.Nil(t, alarmer.Clear(a), "clear failed")
 311
 312         time.Sleep(time.Duration(2) * time.Second)
 313         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 314 }
 315
 316 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 317         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 318         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 319         defer ts.Close()
 320
 321         // Raise two alarms
 322         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 323     assert.Nil(t, alarmer.Raise(a), "raise failed")
 324
 325         b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11")
 326         assert.Nil(t, alarmer.Raise(b), "raise failed")
 327
 328         time.Sleep(time.Duration(5) * time.Second)
 329
 330         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 331         VerifyAlarm(t, a, 1)
 332         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 333         VerifyAlarm(t, b, 2)
 334 }
 335
 336 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 337         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 338         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 339         defer ts.Close()
 340
 341         // Raise two alarms
 342         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 343         assert.Nil(t, alarmer.Clear(a), "clear failed")
 344
 345         b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11")
 346         assert.Nil(t, alarmer.Clear(b), "clear failed")
 347
 348         time.Sleep(time.Duration(2) * time.Second)
 349         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 350 }
 351
 352 func TestAlarmsSuppresedSucess(t *testing.T) {
 353         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 354         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 355         defer ts.Close()
 356
 357         // Raise two similar/matching alarms ... the second one suppresed
 358         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 359         assert.Nil(t, alarmer.Raise(a), "raise failed")
 360         assert.Nil(t, alarmer.Raise(a), "raise failed")
 361
 362         VerifyAlarm(t, a, 1)
 363         assert.Nil(t, alarmer.Clear(a), "clear failed")
 364 }
 365
 366 func TestInvalidAlarms(t *testing.T) {
 367         xapp.Logger.Info("TestInvalidAlarms")
 368         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 369         assert.Nil(t, alarmer.Raise(a), "raise failed")
 370         time.Sleep(time.Duration(2) * time.Second)
 371 }
 372
 373 func TestAlarmHandlingErrorCases(t *testing.T) {
 374         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 375         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 376         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 377         assert.Nil(t, ok, "raise failed")
 378 }
 379
 380 func TestConsumeUnknownMessage(t *testing.T) {
 381         xapp.Logger.Info("TestConsumeUnknownMessage")
 382         err := alarmManager.Consume(&xapp.RMRParams{})
 383         assert.Nil(t, err, "raise failed")
 384 }
 385
 386 func TestStatusCallback(t *testing.T) {
 387         xapp.Logger.Info("TestStatusCallback")
 388         assert.Equal(t, true, alarmManager.StatusCB())
 389 }
 390
 391 func TestActiveAlarmMaxThresholds(t *testing.T) {
 392         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 393         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 394         alarmManager.maxActiveAlarms = 0
 395         alarmManager.maxAlarmHistory = 10
 396
 397         a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 398         assert.Nil(t, alarmer.Raise(a), "raise failed")
 399
 400         var alarmConfigParams alarm.AlarmConfigParams
 401         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 402         req = mux.SetURLVars(req, nil)
 403         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 404         response := executeRequest(req, handleFunc)
 405
 406         // Check HTTP Status Code
 407         checkResponseCode(t, http.StatusOK, response.Code)
 408
 409         // Decode the json output from handler
 410         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 411         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 412                 t.Errorf("Incorrect alarm thresholds")
 413         }
 414
 415         time.Sleep(time.Duration(1) * time.Second)
 416         alarmManager.maxActiveAlarms = 5000
 417         alarmManager.maxAlarmHistory = 20000
 418         VerifyAlarm(t, a, 2)
 419         VerifyAlarm(t, a, 2)
 420         ts.Close()
 421 }
 422
 423 func TestGetPrometheusAlerts(t *testing.T) {
 424         time.Sleep(1 * time.Second)
 425         xapp.Logger.Info("TestGetPrometheusAlerts")
 426         ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
 427
 428         commandReady := make(chan bool, 1)
 429         command := "cli/alarm-cli"
 430         args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
 431         ExecCLICommand(commandReady, command, args...)
 432         <-commandReady
 433
 434         ts.Close()
 435 }
 436
 437 func TestDelayedAlarmRaiseAndClear(t *testing.T) {
 438         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
 439
 440         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 441         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 442
 443         // Add new alarm definition
 444         var alarm9999Definition alarm.AlarmDefinition
 445         alarm9999Definition.AlarmId = 9999
 446         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 447         alarm9999Definition.EventType = "Test type"
 448         alarm9999Definition.OperationInstructions = "Not defined"
 449         alarm9999Definition.RaiseDelay = 1
 450         alarm9999Definition.ClearDelay = 1
 451         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 452         pbodyEn, _ := json.Marshal(pbodyParams)
 453         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 454         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 455         response := executeRequest(req, handleFunc)
 456         checkResponseCode(t, http.StatusOK, response.Code)
 457
 458         // Verify 9999 alarm definition
 459         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 460         vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 461         req = mux.SetURLVars(req, vars)
 462         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 463         response = executeRequest(req, handleFunc)
 464         checkResponseCode(t, http.StatusOK, response.Code)
 465
 466         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 467         defer ts.Close()
 468
 469         // Raise alarm. Posting alert and updating alarm history should be delayed
 470         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 471         assert.Nil(t, alarmer.Raise(a), "raise failed")
 472         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 473
 474         // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
 475         assert.Nil(t, alarmer.Clear(a), "clear failed")
 476
 477         time.Sleep(time.Duration(2) * time.Second)
 478         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 479         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
 480 }
 481
 482 func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
 483         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
 484
 485         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 486         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 487
 488         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 489         defer ts.Close()
 490
 491         // Raise two alarms. The first should be delayed
 492         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 493         assert.Nil(t, alarmer.Raise(a), "raise failed")
 494         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 495
 496         b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 497         assert.Nil(t, alarmer.Raise(b), "raise failed")
 498         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 499
 500         // Clear two alarms. The first should be delayed. Check the alarms are removed
 501         assert.Nil(t, alarmer.Clear(a), "clear failed")
 502         assert.Nil(t, alarmer.Clear(b), "clear failed")
 503
 504         time.Sleep(time.Duration(2) * time.Second)
 505         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 506         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 507 }
 508
 509 func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
 510         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
 511
 512         // Delete exisitng 9999 alarm definition
 513         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 514         vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
 515         req = mux.SetURLVars(req, vars)
 516         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 517         response := executeRequest(req, handleFunc)
 518         checkResponseCode(t, http.StatusOK, response.Code)
 519
 520         // Add updated 9999 alarm definition
 521         var alarm9999Definition alarm.AlarmDefinition
 522         alarm9999Definition.AlarmId = 9999
 523         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 524         alarm9999Definition.EventType = "Test type"
 525         alarm9999Definition.OperationInstructions = "Not defined"
 526         alarm9999Definition.RaiseDelay = 1
 527         alarm9999Definition.ClearDelay = 0
 528         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 529         pbodyEn, _ := json.Marshal(pbodyParams)
 530         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 531         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 532         response = executeRequest(req, handleFunc)
 533         checkResponseCode(t, http.StatusOK, response.Code)
 534
 535         // Verify 9999 alarm definition
 536         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 537         vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)}
 538         req = mux.SetURLVars(req, vars)
 539         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 540         response = executeRequest(req, handleFunc)
 541         checkResponseCode(t, http.StatusOK, response.Code)
 542
 543         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 544         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 545
 546         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 547         defer ts.Close()
 548
 549         // Raise two alarms. The first should be delayed
 550         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 551         assert.Nil(t, alarmer.Raise(a), "raise failed")
 552         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 553
 554         b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1")
 555         assert.Nil(t, alarmer.Raise(b), "raise failed")
 556         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 557
 558         // Clear two alarms. The first should be delayed. Check the alarms are removed
 559         assert.Nil(t, alarmer.Clear(a), "clear failed")
 560         assert.Nil(t, alarmer.Clear(b), "clear failed")
 561
 562         time.Sleep(time.Duration(2) * time.Second)
 563         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 564         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 565 }
 566
 567 func TestClearExpiredAlarms(t *testing.T) {
 568         xapp.Logger.Info("TestClearExpiredAlarms")
 569
 570         a := alarm.AlarmMessage{
 571                 Alarm:       alarmer.NewAlarm(72004, alarm.SeverityWarning, "threshold", ""),
 572                 AlarmAction: alarm.AlarmActionRaise,
 573                 AlarmTime:   time.Now().UnixNano(),
 574         }
 575         d := alarm.RICAlarmDefinitions[72004]
 576         n := AlarmNotification{a, *d}
 577         alarmManager.activeAlarms = make([]AlarmNotification, 0)
 578         alarmManager.UpdateActiveAlarmList(&n)
 579
 580         // Unknown SP
 581         a.Alarm.SpecificProblem = 1234
 582         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 583
 584         // TTL is 0
 585         d.TimeToLive = 0
 586         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 587
 588         // TTL not expired
 589         a.Alarm.SpecificProblem = 72004
 590         d.TimeToLive = 2
 591         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 592
 593         // TTL expired, alarm should be cleared
 594         time.Sleep(time.Duration(3) * time.Second)
 595         assert.Equal(t, len(alarmManager.activeAlarms), 1)
 596         assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 597         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 598 }
 599
 600 func TestSetAlarmConfig(t *testing.T) {
 601         xapp.Logger.Info("TestSetAlarmConfig")
 602
 603         var setAlarmConfig alarm.AlarmConfigParams
 604         setAlarmConfig.MaxActiveAlarms = 500
 605         setAlarmConfig.MaxAlarmHistory = 2000
 606
 607         pbodyEn, _ := json.Marshal(setAlarmConfig)
 608         req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 609         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 610         response := executeRequest(req, handleFunc)
 611         checkResponseCode(t, http.StatusOK, response.Code)
 612
 613         var getAlarmConfig alarm.AlarmConfigParams
 614         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 615         req = mux.SetURLVars(req, nil)
 616         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 617         response = executeRequest(req, handleFunc)
 618         checkResponseCode(t, http.StatusOK, response.Code)
 619
 620         // Decode the json output from handler
 621         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 622         if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 {
 623                 t.Errorf("Incorrect alarm thresholds")
 624         }
 625
 626         // Revert ot default
 627         setAlarmConfig.MaxActiveAlarms = 5000
 628         setAlarmConfig.MaxAlarmHistory = 20000
 629
 630         pbodyEn, _ = json.Marshal(setAlarmConfig)
 631         req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 632         handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig)
 633         response = executeRequest(req, handleFunc)
 634         checkResponseCode(t, http.StatusOK, response.Code)
 635
 636         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 637         req = mux.SetURLVars(req, nil)
 638         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 639         response = executeRequest(req, handleFunc)
 640         checkResponseCode(t, http.StatusOK, response.Code)
 641
 642         // Decode the json output from handler
 643         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 644         if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 {
 645                 t.Errorf("Incorrect alarm thresholds")
 646         }
 647 }
 648
 649 func TestConfigChangeCB(t *testing.T) {
 650         xapp.Logger.Info("TestConfigChangeCB")
 651         alarmManager.ConfigChangeCB("AlarmManager")
 652 }
 653
 654 func TestPersistentStorage(t *testing.T) {
 655     xapp.Logger.Info("TestPersistentStorage")
 656     alarmManager.alarmInfoPvFile = "../../definitions/sample.json"
 657     alarmManager.ReadAlarmInfoFromPersistentVolume()
 658 }
 659
 660 func TestReadAlarmDefinitionFromJsonWrongFilename(t *testing.T) {
 661   // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 662     xapp.Logger.Info("TestReadAlarmDefinitionFromJsonWrongFilename")
 663     os.Setenv("DEF_FILE","test.json")
 664     alarmManager.ReadAlarmDefinitionFromJson()
 665   // correct the filename
 666 }
 667
 668 func TestReadAlarmDefinitionFromJsonInvalidFilename(t *testing.T) {
 669   // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 670     xapp.Logger.Info("TestReadAlarmDefinitionFromJsonInvalidFilename")
 671     os.Setenv("DEF_FILE","../../definitions/test.json")
 672     alarmManager.ReadAlarmDefinitionFromJson()
 673   // correct the filename
 674 }
 675
 676 func TestPostAlarm(t *testing.T) {
 677         xapp.Logger.Info("TestPostAlarm")
 678         var activeAlarms []AlarmNotification
 679         activeAlarms = make([]AlarmNotification, 1)
 680         alarmManager.PostAlarm(&activeAlarms[0])
 681 }
 682
 683 func TestPostAlarm1(t *testing.T) {
 684         xapp.Logger.Info("TestPostAlarm")
 685         var activeAlarms []AlarmNotification
 686         activeAlarms = make([]AlarmNotification, 2)
 687         alarmManager.PostAlarm(&activeAlarms[0])
 688 }
 689
 690 func TestNewAlarmManagerOther(t *testing.T){
 691     NewAlarmManager("", 0, true)
 692 }
 693
 694 func TestStatusCallbackFailure(t *testing.T) {
 695         xapp.Logger.Info("TestStatusCallbackFailure")
 696         alarmManager.rmrReady = false
 697         assert.Equal(t, false, alarmManager.StatusCB())
 698 }
 699
 700 func TestConfigChangeCBFailure(t *testing.T) {
 701         xapp.Logger.Info("TestConfigChangeCBFailure")
 702         alarmManager.maxActiveAlarms = 0
 703         alarmManager.maxAlarmHistory = 0
 704         alarmManager.ConfigChangeCB("AlarmManager")
 705 }
 706
 707
 708 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 709         receivedAlert := waitForEvent()
 710
 711         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 712         _, ok := alarmManager.IsMatchFound(a)
 713         assert.True(t, ok)
 714
 715         return receivedAlert
 716 }
 717
 718 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 719         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 720         //assert.Equal(t, receivedAlert, e)
 721 }
 722
 723 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 724         l, err := net.Listen("tcp", "localhost:9093")
 725         if err != nil {
 726                 t.Error("Failed to create listener: " + err.Error())
 727         }
 728         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 729                 assert.Equal(t, r.Method, method)
 730                 assert.Equal(t, r.URL.String(), url)
 731
 732                 fireEvent(t, r.Body)
 733
 734                 w.Header().Add("Content-Type", "application/json")
 735                 w.WriteHeader(status)
 736                 b, _ := json.Marshal(respData)
 737                 w.Write(b)
 738         }))
 739         ts.Listener.Close()
 740         ts.Listener = l
 741
 742         ts.Start()
 743
 744         return ts
 745 }
 746
 747 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
 748         l, err := net.Listen("tcp", "localhost:9093")
 749         if err != nil {
 750                 t.Error("Failed to create listener: " + err.Error())
 751         }
 752         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 753                 assert.Equal(t, r.Method, method)
 754                 assert.Equal(t, r.URL.String(), url)
 755
 756                 w.Header().Add("Content-Type", "application/json")
 757                 w.WriteHeader(200)
 758                 // Read alerts from file
 759                 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
 760                 if err != nil {
 761                         t.Error("Failed to send response: ", err)
 762                 }
 763                 _, err = w.Write(payload)
 764                 if err != nil {
 765                         t.Error("Failed to send response: " + err.Error())
 766                 }
 767         }))
 768         ts.Listener.Close()
 769         ts.Listener = l
 770         ts.Start()
 771         return ts
 772 }
 773
 774 func waitForEvent() string {
 775         receivedAlert := <-eventChan
 776         return receivedAlert
 777 }
 778
 779 func fireEvent(t *testing.T, body io.ReadCloser) {
 780         reqBody, err := ioutil.ReadAll(body)
 781         assert.Nil(t, err, "ioutil.ReadAll failed")
 782         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 783
 784         eventChan <- fmt.Sprintf("%s", reqBody)
 785 }
 786
 787 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 788         rr := httptest.NewRecorder()
 789
 790         handleR.ServeHTTP(rr, req)
 791
 792         return rr
 793 }
 794
 795 func checkResponseCode(t *testing.T, expected, actual int) bool {
 796         if expected != actual {
 797                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 798                 return false
 799         }
 800         return true
 801 }
 802
 803 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
 804         go func() {
 805                 xapp.Logger.Info("Giving CLI command")
 806                 cmd := exec.Command(command, args...)
 807                 cmd.Dir = "../"
 808                 output, err := cmd.CombinedOutput()
 809                 if err != nil {
 810                         xapp.Logger.Info("CLI command failed out: %s", err)
 811                 }
 812                 xapp.Logger.Info("CLI command output: %s", output)
 813                 commandReady <- true
 814                 xapp.Logger.Info("CLI command completed")
 815         }()
 816 }
 817
 818 func readJSONFromFile(filename string) ([]byte, error) {
 819         file, err := ioutil.ReadFile(filename)
 820         if err != nil {
 821                 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
 822                 return nil, err
 823         }
 824         return file, nil
 825 }
 826
 827