manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io"
  28         "io/ioutil"
  29         "net"
  30         "net/http"
  31         "net/http/httptest"
  32         "os"
  33         "os/exec"
  34         "strconv"
  35         "strings"
  36         "testing"
  37         "time"
  38
  39         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  40         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  41         "github.com/gorilla/mux"
  42         "github.com/prometheus/alertmanager/api/v2/models"
  43         "github.com/stretchr/testify/assert"
  44 )
  45
  46 var alarmManager *AlarmManager
  47 var alarmer *alarm.RICAlarm
  48 var eventChan chan string
  49
  50 // Test cases
  51 func TestMain(M *testing.M) {
  52         alarmManager = NewAlarmManager("localhost:9093", 500, false)
  53         alarmManager.alertInterval = 20000
  54         go alarmManager.Run(false, 5)
  55         time.Sleep(time.Duration(10) * time.Second)
  56
  57         // Wait until RMR is up-and-running
  58         for !xapp.Rmr.IsReady() {
  59                 time.Sleep(time.Duration(1) * time.Second)
  60         }
  61
  62         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  63         alarmManager.alarmClient = alarmer
  64         time.Sleep(time.Duration(5) * time.Second)
  65         eventChan = make(chan string)
  66
  67         os.Exit(M.Run())
  68 }
  69
  70 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  71         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  72         var alarmDefinition alarm.AlarmDefinition
  73         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  74         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
  75         req = mux.SetURLVars(req, vars)
  76         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  77         response := executeRequest(req, handleFunc)
  78         checkResponseCode(t, http.StatusOK, response.Code)
  79         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  80         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  81         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
  82                 t.Errorf("Incorrect alarm definition")
  83         }
  84 }
  85
  86 func TestSetAlarmDefinitions(t *testing.T) {
  87         xapp.Logger.Info("TestSetAlarmDefinitions")
  88         var alarm8004Definition alarm.AlarmDefinition
  89         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  90         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  91         alarm8004Definition.EventType = "Processing error"
  92         alarm8004Definition.OperationInstructions = "Not defined"
  93         alarm8004Definition.RaiseDelay = 0
  94         alarm8004Definition.ClearDelay = 0
  95
  96         var alarm8005Definition alarm.AlarmDefinition
  97         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  98         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  99         alarm8005Definition.EventType = "Communication error"
 100         alarm8005Definition.OperationInstructions = "Not defined"
 101         alarm8005Definition.RaiseDelay = 0
 102         alarm8005Definition.ClearDelay = 0
 103
 104         var alarm8006Definition alarm.AlarmDefinition
 105         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
 106         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 107         alarm8006Definition.EventType = "Communication error"
 108         alarm8006Definition.OperationInstructions = "Not defined"
 109         alarm8006Definition.RaiseDelay = 0
 110         alarm8006Definition.ClearDelay = 0
 111
 112         var alarm8007Definition alarm.AlarmDefinition
 113         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 114         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 115         alarm8007Definition.EventType = "Communication error"
 116         alarm8007Definition.OperationInstructions = "Not defined"
 117         alarm8007Definition.RaiseDelay = 0
 118         alarm8007Definition.ClearDelay = 0
 119
 120         var alarm8008Definition alarm.AlarmDefinition
 121         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 122         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 123         alarm8008Definition.EventType = "storage warning"
 124         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
 125         alarm8008Definition.RaiseDelay = 0
 126         alarm8008Definition.ClearDelay = 0
 127
 128         var alarm8009Definition alarm.AlarmDefinition
 129         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 130         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 131         alarm8009Definition.EventType = "storage warning"
 132         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
 133         alarm8009Definition.RaiseDelay = 0
 134         alarm8009Definition.ClearDelay = 0
 135
 136         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 137         pbodyEn, _ := json.Marshal(pbodyParams)
 138         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 139         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 140         response := executeRequest(req, handleFunc)
 141         status := checkResponseCode(t, http.StatusOK, response.Code)
 142         xapp.Logger.Info("status = %v", status)
 143
 144 }
 145
 146 func TestGetAlarmDefinitions(t *testing.T) {
 147         xapp.Logger.Info("TestGetAlarmDefinitions")
 148         var alarmDefinition alarm.AlarmDefinition
 149         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 150         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 151         req = mux.SetURLVars(req, vars)
 152         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 153         response := executeRequest(req, handleFunc)
 154         checkResponseCode(t, http.StatusOK, response.Code)
 155         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 156         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 157         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 158                 t.Errorf("Incorrect alarm definition")
 159         }
 160 }
 161
 162 func TestDeleteAlarmDefinitions(t *testing.T) {
 163         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 164         //Get all
 165         var ricAlarmDefinitions RicAlarmDefinitions
 166         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 167         req = mux.SetURLVars(req, nil)
 168         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 169         response := executeRequest(req, handleFunc)
 170         checkResponseCode(t, http.StatusOK, response.Code)
 171         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 172         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 173                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 174         }
 175
 176         //Delete 8004
 177         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 178         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 179         req = mux.SetURLVars(req, vars)
 180         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 181         response = executeRequest(req, handleFunc)
 182         checkResponseCode(t, http.StatusOK, response.Code)
 183
 184         //Get 8004 fail
 185         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 186         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 187         req = mux.SetURLVars(req, vars)
 188         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 189         response = executeRequest(req, handleFunc)
 190         checkResponseCode(t, http.StatusBadRequest, response.Code)
 191
 192         //Set 8004 success
 193         var alarm8004Definition alarm.AlarmDefinition
 194         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 195         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 196         alarm8004Definition.EventType = "Processing error"
 197         alarm8004Definition.OperationInstructions = "Not defined"
 198         alarm8004Definition.RaiseDelay = 0
 199         alarm8004Definition.ClearDelay = 0
 200         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 201         pbodyEn, _ := json.Marshal(pbodyParams)
 202         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 203         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 204         response = executeRequest(req, handleFunc)
 205         checkResponseCode(t, http.StatusOK, response.Code)
 206
 207         //Get 8004 success
 208         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 209         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 210         req = mux.SetURLVars(req, vars)
 211         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 212         response = executeRequest(req, handleFunc)
 213         checkResponseCode(t, http.StatusOK, response.Code)
 214 }
 215
 216 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 217         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 218         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 219         defer ts.Close()
 220
 221         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 222         assert.Nil(t, alarmer.Raise(a), "raise failed")
 223
 224         VerifyAlarm(t, a, 1)
 225
 226         var activeAlarms []AlarmNotification
 227         activeAlarms = make([]AlarmNotification, 1)
 228         req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil)
 229         req = mux.SetURLVars(req, nil)
 230         handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms)
 231         response := executeRequest(req, handleFunc)
 232         checkResponseCode(t, http.StatusOK, response.Code)
 233
 234         // Decode the json output from handler
 235         json.NewDecoder(response.Body).Decode(activeAlarms)
 236         if len(activeAlarms) != 1 {
 237                 t.Errorf("Incorrect alarm alarm count")
 238         }
 239
 240         var alarmHistory []AlarmNotification
 241         alarmHistory = make([]AlarmNotification, 1)
 242         req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil)
 243         req = mux.SetURLVars(req, nil)
 244         handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory)
 245         response = executeRequest(req, handleFunc)
 246         checkResponseCode(t, http.StatusOK, response.Code)
 247
 248         // Decode the json output from handler
 249         json.NewDecoder(response.Body).Decode(alarmHistory)
 250         if len(alarmHistory) != 1 {
 251                 t.Errorf("Incorrect alarm history count")
 252         }
 253 }
 254
 255 func TestAlarmClearedSucess(t *testing.T) {
 256         xapp.Logger.Info("TestAlarmClearedSucess")
 257         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 258         defer ts.Close()
 259
 260         // Raise the alarm
 261         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 262         assert.Nil(t, alarmer.Raise(a), "raise failed")
 263
 264         VerifyAlarm(t, a, 1)
 265
 266         // Now Clear the alarm and check alarm is removed
 267         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 268         assert.Nil(t, alarmer.Clear(a), "clear failed")
 269
 270         time.Sleep(time.Duration(2) * time.Second)
 271         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 272 }
 273
 274 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 275         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 276         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 277         defer ts.Close()
 278
 279         // Raise two alarms
 280         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 281         assert.Nil(t, alarmer.Raise(a), "raise failed")
 282
 283         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 284         assert.Nil(t, alarmer.Raise(b), "raise failed")
 285
 286         time.Sleep(time.Duration(5) * time.Second)
 287
 288         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 289         VerifyAlarm(t, a, 1)
 290         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 291         VerifyAlarm(t, b, 2)
 292 }
 293
 294 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 295         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 296         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 297         defer ts.Close()
 298
 299         // Raise two alarms
 300         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 301         assert.Nil(t, alarmer.Clear(a), "clear failed")
 302
 303         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 304         assert.Nil(t, alarmer.Clear(b), "clear failed")
 305
 306         time.Sleep(time.Duration(2) * time.Second)
 307         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 308 }
 309
 310 func TestAlarmsSuppresedSucess(t *testing.T) {
 311         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 312         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 313         defer ts.Close()
 314
 315         // Raise two similar/matching alarms ... the second one suppresed
 316         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 317         assert.Nil(t, alarmer.Raise(a), "raise failed")
 318         assert.Nil(t, alarmer.Raise(a), "raise failed")
 319
 320         VerifyAlarm(t, a, 1)
 321         assert.Nil(t, alarmer.Clear(a), "clear failed")
 322 }
 323
 324 func TestInvalidAlarms(t *testing.T) {
 325         xapp.Logger.Info("TestInvalidAlarms")
 326         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 327         assert.Nil(t, alarmer.Raise(a), "raise failed")
 328         time.Sleep(time.Duration(2) * time.Second)
 329 }
 330
 331 func TestAlarmHandlingErrorCases(t *testing.T) {
 332         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 333         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 334         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 335         assert.Nil(t, ok, "raise failed")
 336 }
 337
 338 func TestConsumeUnknownMessage(t *testing.T) {
 339         xapp.Logger.Info("TestConsumeUnknownMessage")
 340         err := alarmManager.Consume(&xapp.RMRParams{})
 341         assert.Nil(t, err, "raise failed")
 342 }
 343
 344 func TestStatusCallback(t *testing.T) {
 345         xapp.Logger.Info("TestStatusCallback")
 346         assert.Equal(t, true, alarmManager.StatusCB())
 347 }
 348
 349 func TestActiveAlarmMaxThresholds(t *testing.T) {
 350         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 351         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 352         alarmManager.maxActiveAlarms = 0
 353         alarmManager.maxAlarmHistory = 10
 354
 355         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 356         assert.Nil(t, alarmer.Raise(a), "raise failed")
 357
 358         var alarmConfigParams alarm.AlarmConfigParams
 359         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 360         req = mux.SetURLVars(req, nil)
 361         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 362         response := executeRequest(req, handleFunc)
 363
 364         // Check HTTP Status Code
 365         checkResponseCode(t, http.StatusOK, response.Code)
 366
 367         // Decode the json output from handler
 368         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 369         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 370                 t.Errorf("Incorrect alarm thresholds")
 371         }
 372
 373         time.Sleep(time.Duration(1) * time.Second)
 374         alarmManager.maxActiveAlarms = 5000
 375         alarmManager.maxAlarmHistory = 20000
 376         VerifyAlarm(t, a, 2)
 377         VerifyAlarm(t, a, 2)
 378         ts.Close()
 379 }
 380
 381 func TestGetPrometheusAlerts(t *testing.T) {
 382         time.Sleep(1 * time.Second)
 383         xapp.Logger.Info("TestGetPrometheusAlerts")
 384         ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
 385
 386         commandReady := make(chan bool, 1)
 387         command := "cli/alarm-cli"
 388         args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
 389         ExecCLICommand(commandReady, command, args...)
 390         <-commandReady
 391
 392         ts.Close()
 393 }
 394
 395 func TestDelayedAlarmRaiseAndClear(t *testing.T) {
 396         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
 397
 398         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 399         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 400
 401         // Add new alarm definition
 402         var alarm9999Definition alarm.AlarmDefinition
 403         alarm9999Definition.AlarmId = 9999
 404         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 405         alarm9999Definition.EventType = "Test type"
 406         alarm9999Definition.OperationInstructions = "Not defined"
 407         alarm9999Definition.RaiseDelay = 1
 408         alarm9999Definition.ClearDelay = 1
 409         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 410         pbodyEn, _ := json.Marshal(pbodyParams)
 411         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 412         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 413         response := executeRequest(req, handleFunc)
 414         checkResponseCode(t, http.StatusOK, response.Code)
 415
 416         // Verify 9999 alarm definition
 417         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 418         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 419         req = mux.SetURLVars(req, vars)
 420         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 421         response = executeRequest(req, handleFunc)
 422         checkResponseCode(t, http.StatusOK, response.Code)
 423
 424         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 425         defer ts.Close()
 426
 427         // Raise alarm. Posting alert and updating alarm history should be delayed
 428         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 429         assert.Nil(t, alarmer.Raise(a), "raise failed")
 430         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 431
 432         // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
 433         assert.Nil(t, alarmer.Clear(a), "clear failed")
 434
 435         time.Sleep(time.Duration(2) * time.Second)
 436         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 437         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
 438 }
 439
 440 func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
 441         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
 442
 443         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 444         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 445
 446         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 447         defer ts.Close()
 448
 449         // Raise two alarms. The first should be delayed
 450         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 451         assert.Nil(t, alarmer.Raise(a), "raise failed")
 452         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 453
 454         b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 455         assert.Nil(t, alarmer.Raise(b), "raise failed")
 456         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 457
 458         // Clear two alarms. The first should be delayed. Check the alarms are removed
 459         assert.Nil(t, alarmer.Clear(a), "clear failed")
 460         assert.Nil(t, alarmer.Clear(b), "clear failed")
 461
 462         time.Sleep(time.Duration(2) * time.Second)
 463         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 464         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 465 }
 466
 467 func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
 468         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
 469
 470         // Delete exisitng 9999 alarm definition
 471         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 472         vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
 473         req = mux.SetURLVars(req, vars)
 474         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 475         response := executeRequest(req, handleFunc)
 476         checkResponseCode(t, http.StatusOK, response.Code)
 477
 478         // Add updated 9999 alarm definition
 479         var alarm9999Definition alarm.AlarmDefinition
 480         alarm9999Definition.AlarmId = 9999
 481         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 482         alarm9999Definition.EventType = "Test type"
 483         alarm9999Definition.OperationInstructions = "Not defined"
 484         alarm9999Definition.RaiseDelay = 1
 485         alarm9999Definition.ClearDelay = 0
 486         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 487         pbodyEn, _ := json.Marshal(pbodyParams)
 488         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 489         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 490         response = executeRequest(req, handleFunc)
 491         checkResponseCode(t, http.StatusOK, response.Code)
 492
 493         // Verify 9999 alarm definition
 494         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 495         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 496         req = mux.SetURLVars(req, vars)
 497         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 498         response = executeRequest(req, handleFunc)
 499         checkResponseCode(t, http.StatusOK, response.Code)
 500
 501         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 502         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 503
 504         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 505         defer ts.Close()
 506
 507         // Raise two alarms. The first should be delayed
 508         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 509         assert.Nil(t, alarmer.Raise(a), "raise failed")
 510         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 511
 512         b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 513         assert.Nil(t, alarmer.Raise(b), "raise failed")
 514         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 515
 516         // Clear two alarms. The first should be delayed. Check the alarms are removed
 517         assert.Nil(t, alarmer.Clear(a), "clear failed")
 518         assert.Nil(t, alarmer.Clear(b), "clear failed")
 519
 520         time.Sleep(time.Duration(2) * time.Second)
 521         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 522         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 523 }
 524
 525 func TestClearExpiredAlarms(t *testing.T) {
 526         xapp.Logger.Info("TestClearExpiredAlarms")
 527
 528         a := alarm.AlarmMessage{
 529                 Alarm:       alarmer.NewAlarm(8007, alarm.SeverityWarning, "threshold", ""),
 530                 AlarmAction: alarm.AlarmActionRaise,
 531                 AlarmTime:   time.Now().UnixNano(),
 532         }
 533         d := alarm.RICAlarmDefinitions[8007]
 534         n := AlarmNotification{a, *d}
 535         alarmManager.activeAlarms = make([]AlarmNotification, 0)
 536         alarmManager.UpdateActiveAlarmList(&n)
 537
 538         // Unknown SP
 539         a.Alarm.SpecificProblem = 1234
 540         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 541
 542         // TTL is 0
 543         d.TimeToLive = 0
 544         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 545
 546         // TTL not expired
 547         a.Alarm.SpecificProblem = 8007
 548         d.TimeToLive = 2
 549         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 550
 551         // TTL expired, alarm should be cleared
 552         time.Sleep(time.Duration(3) * time.Second)
 553         assert.Equal(t, len(alarmManager.activeAlarms), 1)
 554         assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 555         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 556 }
 557
 558 func TestSetAlarmConfig(t *testing.T) {
 559         xapp.Logger.Info("TestSetAlarmConfig")
 560
 561         var setAlarmConfig alarm.AlarmConfigParams
 562         setAlarmConfig.MaxActiveAlarms = 500
 563         setAlarmConfig.MaxAlarmHistory = 2000
 564
 565         pbodyEn, _ := json.Marshal(setAlarmConfig)
 566         req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 567         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 568         response := executeRequest(req, handleFunc)
 569         checkResponseCode(t, http.StatusOK, response.Code)
 570
 571         var getAlarmConfig alarm.AlarmConfigParams
 572         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 573         req = mux.SetURLVars(req, nil)
 574         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 575         response = executeRequest(req, handleFunc)
 576         checkResponseCode(t, http.StatusOK, response.Code)
 577
 578         // Decode the json output from handler
 579         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 580         if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 {
 581                 t.Errorf("Incorrect alarm thresholds")
 582         }
 583
 584         // Revert ot default
 585         setAlarmConfig.MaxActiveAlarms = 5000
 586         setAlarmConfig.MaxAlarmHistory = 20000
 587
 588         pbodyEn, _ = json.Marshal(setAlarmConfig)
 589         req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 590         handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig)
 591         response = executeRequest(req, handleFunc)
 592         checkResponseCode(t, http.StatusOK, response.Code)
 593
 594         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 595         req = mux.SetURLVars(req, nil)
 596         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 597         response = executeRequest(req, handleFunc)
 598         checkResponseCode(t, http.StatusOK, response.Code)
 599
 600         // Decode the json output from handler
 601         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 602         if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 {
 603                 t.Errorf("Incorrect alarm thresholds")
 604         }
 605 }
 606
 607 func TestConfigChangeCB(t *testing.T) {
 608         xapp.Logger.Info("TestConfigChangeCB")
 609         alarmManager.ConfigChangeCB("AlarmManager")
 610 }
 611
 612 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 613         receivedAlert := waitForEvent()
 614
 615         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 616         _, ok := alarmManager.IsMatchFound(a)
 617         assert.True(t, ok)
 618
 619         return receivedAlert
 620 }
 621
 622 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 623         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 624         //assert.Equal(t, receivedAlert, e)
 625 }
 626
 627 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 628         l, err := net.Listen("tcp", "localhost:9093")
 629         if err != nil {
 630                 t.Error("Failed to create listener: " + err.Error())
 631         }
 632         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 633                 assert.Equal(t, r.Method, method)
 634                 assert.Equal(t, r.URL.String(), url)
 635
 636                 fireEvent(t, r.Body)
 637
 638                 w.Header().Add("Content-Type", "application/json")
 639                 w.WriteHeader(status)
 640                 b, _ := json.Marshal(respData)
 641                 w.Write(b)
 642         }))
 643         ts.Listener.Close()
 644         ts.Listener = l
 645
 646         ts.Start()
 647
 648         return ts
 649 }
 650
 651 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
 652         l, err := net.Listen("tcp", "localhost:9093")
 653         if err != nil {
 654                 t.Error("Failed to create listener: " + err.Error())
 655         }
 656         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 657                 assert.Equal(t, r.Method, method)
 658                 assert.Equal(t, r.URL.String(), url)
 659
 660                 w.Header().Add("Content-Type", "application/json")
 661                 w.WriteHeader(200)
 662                 // Read alerts from file
 663                 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
 664                 if err != nil {
 665                         t.Error("Failed to send response: ", err)
 666                 }
 667                 _, err = w.Write(payload)
 668                 if err != nil {
 669                         t.Error("Failed to send response: " + err.Error())
 670                 }
 671         }))
 672         ts.Listener.Close()
 673         ts.Listener = l
 674         ts.Start()
 675         return ts
 676 }
 677
 678 func waitForEvent() string {
 679         receivedAlert := <-eventChan
 680         return receivedAlert
 681 }
 682
 683 func fireEvent(t *testing.T, body io.ReadCloser) {
 684         reqBody, err := ioutil.ReadAll(body)
 685         assert.Nil(t, err, "ioutil.ReadAll failed")
 686         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 687
 688         eventChan <- fmt.Sprintf("%s", reqBody)
 689 }
 690
 691 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 692         rr := httptest.NewRecorder()
 693
 694         handleR.ServeHTTP(rr, req)
 695
 696         return rr
 697 }
 698
 699 func checkResponseCode(t *testing.T, expected, actual int) bool {
 700         if expected != actual {
 701                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 702                 return false
 703         }
 704         return true
 705 }
 706
 707 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
 708         go func() {
 709                 xapp.Logger.Info("Giving CLI command")
 710                 cmd := exec.Command(command, args...)
 711                 cmd.Dir = "../"
 712                 output, err := cmd.CombinedOutput()
 713                 if err != nil {
 714                         xapp.Logger.Info("CLI command failed out: %s", err)
 715                 }
 716                 xapp.Logger.Info("CLI command output: %s", output)
 717                 commandReady <- true
 718                 xapp.Logger.Info("CLI command completed")
 719         }()
 720 }
 721
 722 func readJSONFromFile(filename string) ([]byte, error) {
 723         file, err := ioutil.ReadFile(filename)
 724         if err != nil {
 725                 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
 726                 return nil, err
 727         }
 728         return file, nil
 729 }