manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  28         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  29         "github.com/gorilla/mux"
  30         "github.com/prometheus/alertmanager/api/v2/models"
  31         "github.com/stretchr/testify/assert"
  32         "io"
  33         "io/ioutil"
  34         "net"
  35         "net/http"
  36         "net/http/httptest"
  37         "os"
  38         "os/exec"
  39         "strconv"
  40         "strings"
  41         "testing"
  42         "time"
  43 )
  44
  45 var alarmManager *AlarmManager
  46 var alarmer *alarm.RICAlarm
  47 var eventChan chan string
  48
  49 // Test cases
  50 func TestMain(M *testing.M) {
  51         alarmManager = NewAlarmManager("localhost:9093", 500, false)
  52         alarmManager.alertInterval = 20000
  53         go alarmManager.Run(false, 5)
  54         time.Sleep(time.Duration(10) * time.Second)
  55
  56         // Wait until RMR is up-and-running
  57         for !xapp.Rmr.IsReady() {
  58                 time.Sleep(time.Duration(1) * time.Second)
  59         }
  60
  61         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  62         alarmManager.alarmClient = alarmer
  63         time.Sleep(time.Duration(5) * time.Second)
  64         eventChan = make(chan string)
  65
  66         os.Exit(M.Run())
  67 }
  68
  69 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  70         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  71         var alarmDefinition alarm.AlarmDefinition
  72         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  73         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
  74         req = mux.SetURLVars(req, vars)
  75         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  76         response := executeRequest(req, handleFunc)
  77         checkResponseCode(t, http.StatusOK, response.Code)
  78         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  79         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  80         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
  81                 t.Errorf("Incorrect alarm definition")
  82         }
  83 }
  84
  85 func TestSetAlarmDefinitions(t *testing.T) {
  86         xapp.Logger.Info("TestSetAlarmDefinitions")
  87         var alarm8004Definition alarm.AlarmDefinition
  88         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  89         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  90         alarm8004Definition.EventType = "Processing error"
  91         alarm8004Definition.OperationInstructions = "Not defined"
  92         alarm8004Definition.RaiseDelay = 0
  93         alarm8004Definition.ClearDelay = 0
  94
  95         var alarm8005Definition alarm.AlarmDefinition
  96         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  97         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  98         alarm8005Definition.EventType = "Communication error"
  99         alarm8005Definition.OperationInstructions = "Not defined"
 100         alarm8005Definition.RaiseDelay = 0
 101         alarm8005Definition.ClearDelay = 0
 102
 103         var alarm8006Definition alarm.AlarmDefinition
 104         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
 105         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 106         alarm8006Definition.EventType = "Communication error"
 107         alarm8006Definition.OperationInstructions = "Not defined"
 108         alarm8006Definition.RaiseDelay = 0
 109         alarm8006Definition.ClearDelay = 0
 110
 111         var alarm8007Definition alarm.AlarmDefinition
 112         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 113         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 114         alarm8007Definition.EventType = "Communication error"
 115         alarm8007Definition.OperationInstructions = "Not defined"
 116         alarm8007Definition.RaiseDelay = 0
 117         alarm8007Definition.ClearDelay = 0
 118
 119         var alarm8008Definition alarm.AlarmDefinition
 120         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 121         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 122         alarm8008Definition.EventType = "storage warning"
 123         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
 124         alarm8008Definition.RaiseDelay = 0
 125         alarm8008Definition.ClearDelay = 0
 126
 127         var alarm8009Definition alarm.AlarmDefinition
 128         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 129         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 130         alarm8009Definition.EventType = "storage warning"
 131         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
 132         alarm8009Definition.RaiseDelay = 0
 133         alarm8009Definition.ClearDelay = 0
 134
 135         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 136         pbodyEn, _ := json.Marshal(pbodyParams)
 137         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 138         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 139         response := executeRequest(req, handleFunc)
 140         status := checkResponseCode(t, http.StatusOK, response.Code)
 141         xapp.Logger.Info("status = %v", status)
 142
 143 }
 144
 145 func TestGetAlarmDefinitions(t *testing.T) {
 146         xapp.Logger.Info("TestGetAlarmDefinitions")
 147         var alarmDefinition alarm.AlarmDefinition
 148         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 149         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 150         req = mux.SetURLVars(req, vars)
 151         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 152         response := executeRequest(req, handleFunc)
 153         checkResponseCode(t, http.StatusOK, response.Code)
 154         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 155         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 156         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 157                 t.Errorf("Incorrect alarm definition")
 158         }
 159 }
 160
 161 func TestDeleteAlarmDefinitions(t *testing.T) {
 162         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 163         //Get all
 164         var ricAlarmDefinitions RicAlarmDefinitions
 165         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 166         req = mux.SetURLVars(req, nil)
 167         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 168         response := executeRequest(req, handleFunc)
 169         checkResponseCode(t, http.StatusOK, response.Code)
 170         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 171         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 172                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 173         }
 174
 175         //Delete 8004
 176         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 177         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 178         req = mux.SetURLVars(req, vars)
 179         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 180         response = executeRequest(req, handleFunc)
 181         checkResponseCode(t, http.StatusOK, response.Code)
 182
 183         //Get 8004 fail
 184         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 185         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 186         req = mux.SetURLVars(req, vars)
 187         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 188         response = executeRequest(req, handleFunc)
 189         checkResponseCode(t, http.StatusBadRequest, response.Code)
 190
 191         //Set 8004 success
 192         var alarm8004Definition alarm.AlarmDefinition
 193         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 194         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 195         alarm8004Definition.EventType = "Processing error"
 196         alarm8004Definition.OperationInstructions = "Not defined"
 197         alarm8004Definition.RaiseDelay = 0
 198         alarm8004Definition.ClearDelay = 0
 199         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 200         pbodyEn, _ := json.Marshal(pbodyParams)
 201         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 202         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 203         response = executeRequest(req, handleFunc)
 204         checkResponseCode(t, http.StatusOK, response.Code)
 205
 206         //Get 8004 success
 207         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 208         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 209         req = mux.SetURLVars(req, vars)
 210         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 211         response = executeRequest(req, handleFunc)
 212         checkResponseCode(t, http.StatusOK, response.Code)
 213 }
 214
 215 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 216         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 217         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 218         defer ts.Close()
 219
 220         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 221         assert.Nil(t, alarmer.Raise(a), "raise failed")
 222
 223         VerifyAlarm(t, a, 1)
 224
 225         var activeAlarms []AlarmNotification
 226         activeAlarms = make([]AlarmNotification, 1)
 227         req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil)
 228         req = mux.SetURLVars(req, nil)
 229         handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms)
 230         response := executeRequest(req, handleFunc)
 231         checkResponseCode(t, http.StatusOK, response.Code)
 232
 233         // Decode the json output from handler
 234         json.NewDecoder(response.Body).Decode(activeAlarms)
 235         if len(activeAlarms) != 1 {
 236                 t.Errorf("Incorrect alarm alarm count")
 237         }
 238
 239         var alarmHistory []AlarmNotification
 240         alarmHistory = make([]AlarmNotification, 1)
 241         req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil)
 242         req = mux.SetURLVars(req, nil)
 243         handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory)
 244         response = executeRequest(req, handleFunc)
 245         checkResponseCode(t, http.StatusOK, response.Code)
 246
 247         // Decode the json output from handler
 248         json.NewDecoder(response.Body).Decode(alarmHistory)
 249         if len(alarmHistory) != 1 {
 250                 t.Errorf("Incorrect alarm history count")
 251         }
 252 }
 253
 254 func TestAlarmClearedSucess(t *testing.T) {
 255         xapp.Logger.Info("TestAlarmClearedSucess")
 256         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 257         defer ts.Close()
 258
 259         // Raise the alarm
 260         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 261         assert.Nil(t, alarmer.Raise(a), "raise failed")
 262
 263         VerifyAlarm(t, a, 1)
 264
 265         // Now Clear the alarm and check alarm is removed
 266         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 267         assert.Nil(t, alarmer.Clear(a), "clear failed")
 268
 269         time.Sleep(time.Duration(2) * time.Second)
 270         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 271 }
 272
 273 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 274         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 275         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 276         defer ts.Close()
 277
 278         // Raise two alarms
 279         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 280         assert.Nil(t, alarmer.Raise(a), "raise failed")
 281
 282         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 283         assert.Nil(t, alarmer.Raise(b), "raise failed")
 284
 285         time.Sleep(time.Duration(2) * time.Second)
 286         VerifyAlarm(t, a, 2)
 287         VerifyAlarm(t, b, 2)
 288 }
 289
 290 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 291         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 292         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 293         defer ts.Close()
 294
 295         // Raise two alarms
 296         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 297         assert.Nil(t, alarmer.Clear(a), "clear failed")
 298
 299         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 300         assert.Nil(t, alarmer.Clear(b), "clear failed")
 301
 302         time.Sleep(time.Duration(2) * time.Second)
 303         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 304 }
 305
 306 func TestAlarmsSuppresedSucess(t *testing.T) {
 307         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 308         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 309         defer ts.Close()
 310
 311         // Raise two similar/matching alarms ... the second one suppresed
 312         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 313         assert.Nil(t, alarmer.Raise(a), "raise failed")
 314         assert.Nil(t, alarmer.Raise(a), "raise failed")
 315
 316         VerifyAlarm(t, a, 1)
 317         assert.Nil(t, alarmer.Clear(a), "clear failed")
 318 }
 319
 320 func TestInvalidAlarms(t *testing.T) {
 321         xapp.Logger.Info("TestInvalidAlarms")
 322         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 323         assert.Nil(t, alarmer.Raise(a), "raise failed")
 324         time.Sleep(time.Duration(2) * time.Second)
 325 }
 326
 327 func TestAlarmHandlingErrorCases(t *testing.T) {
 328         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 329         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 330         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 331         assert.Nil(t, ok, "raise failed")
 332 }
 333
 334 func TestConsumeUnknownMessage(t *testing.T) {
 335         xapp.Logger.Info("TestConsumeUnknownMessage")
 336         err := alarmManager.Consume(&xapp.RMRParams{})
 337         assert.Nil(t, err, "raise failed")
 338 }
 339
 340 func TestStatusCallback(t *testing.T) {
 341         xapp.Logger.Info("TestStatusCallback")
 342         assert.Equal(t, true, alarmManager.StatusCB())
 343 }
 344
 345 func TestActiveAlarmMaxThresholds(t *testing.T) {
 346         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 347         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 348         alarmManager.maxActiveAlarms = 0
 349         alarmManager.maxAlarmHistory = 10
 350
 351         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 352         assert.Nil(t, alarmer.Raise(a), "raise failed")
 353
 354         var alarmConfigParams alarm.AlarmConfigParams
 355         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 356         req = mux.SetURLVars(req, nil)
 357         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 358         response := executeRequest(req, handleFunc)
 359
 360         // Check HTTP Status Code
 361         checkResponseCode(t, http.StatusOK, response.Code)
 362
 363         // Decode the json output from handler
 364         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 365         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 366                 t.Errorf("Incorrect alarm thresholds")
 367         }
 368
 369         time.Sleep(time.Duration(1) * time.Second)
 370         alarmManager.maxActiveAlarms = 5000
 371         alarmManager.maxAlarmHistory = 20000
 372         VerifyAlarm(t, a, 2)
 373         VerifyAlarm(t, a, 2)
 374         ts.Close()
 375 }
 376
 377 func TestGetPrometheusAlerts(t *testing.T) {
 378         time.Sleep(1 * time.Second)
 379         xapp.Logger.Info("TestGetPrometheusAlerts")
 380         ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
 381
 382         commandReady := make(chan bool, 1)
 383         command := "cli/alarm-cli"
 384         args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
 385         ExecCLICommand(commandReady, command, args...)
 386         <-commandReady
 387
 388         ts.Close()
 389 }
 390
 391 func TestDelayedAlarmRaiseAndClear(t *testing.T) {
 392         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
 393
 394         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 395         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 396
 397         // Add new alarm definition
 398         var alarm9999Definition alarm.AlarmDefinition
 399         alarm9999Definition.AlarmId = 9999
 400         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 401         alarm9999Definition.EventType = "Test type"
 402         alarm9999Definition.OperationInstructions = "Not defined"
 403         alarm9999Definition.RaiseDelay = 1
 404         alarm9999Definition.ClearDelay = 1
 405         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 406         pbodyEn, _ := json.Marshal(pbodyParams)
 407         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 408         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 409         response := executeRequest(req, handleFunc)
 410         checkResponseCode(t, http.StatusOK, response.Code)
 411
 412         // Verify 9999 alarm definition
 413         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 414         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 415         req = mux.SetURLVars(req, vars)
 416         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 417         response = executeRequest(req, handleFunc)
 418         checkResponseCode(t, http.StatusOK, response.Code)
 419
 420         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 421         defer ts.Close()
 422
 423         // Raise alarm. Posting alert and updating alarm history should be delayed
 424         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 425         assert.Nil(t, alarmer.Raise(a), "raise failed")
 426         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 427
 428         // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
 429         assert.Nil(t, alarmer.Clear(a), "clear failed")
 430
 431         time.Sleep(time.Duration(2) * time.Second)
 432         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 433         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
 434 }
 435
 436 func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
 437         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
 438
 439         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 440         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 441
 442         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 443         defer ts.Close()
 444
 445         // Raise two alarms. The first should be delayed
 446         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 447         assert.Nil(t, alarmer.Raise(a), "raise failed")
 448         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 449
 450         b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 451         assert.Nil(t, alarmer.Raise(b), "raise failed")
 452         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 453
 454         // Clear two alarms. The first should be delayed. Check the alarms are removed
 455         assert.Nil(t, alarmer.Clear(a), "clear failed")
 456         assert.Nil(t, alarmer.Clear(b), "clear failed")
 457
 458         time.Sleep(time.Duration(2) * time.Second)
 459         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 460         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 461 }
 462
 463 func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
 464         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
 465
 466         // Delete exisitng 9999 alarm definition
 467         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 468         vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
 469         req = mux.SetURLVars(req, vars)
 470         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 471         response := executeRequest(req, handleFunc)
 472         checkResponseCode(t, http.StatusOK, response.Code)
 473
 474         // Add updated 9999 alarm definition
 475         var alarm9999Definition alarm.AlarmDefinition
 476         alarm9999Definition.AlarmId = 9999
 477         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 478         alarm9999Definition.EventType = "Test type"
 479         alarm9999Definition.OperationInstructions = "Not defined"
 480         alarm9999Definition.RaiseDelay = 1
 481         alarm9999Definition.ClearDelay = 0
 482         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 483         pbodyEn, _ := json.Marshal(pbodyParams)
 484         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 485         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 486         response = executeRequest(req, handleFunc)
 487         checkResponseCode(t, http.StatusOK, response.Code)
 488
 489         // Verify 9999 alarm definition
 490         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 491         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 492         req = mux.SetURLVars(req, vars)
 493         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 494         response = executeRequest(req, handleFunc)
 495         checkResponseCode(t, http.StatusOK, response.Code)
 496
 497         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 498         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 499
 500         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 501         defer ts.Close()
 502
 503         // Raise two alarms. The first should be delayed
 504         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 505         assert.Nil(t, alarmer.Raise(a), "raise failed")
 506         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 507
 508         b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 509         assert.Nil(t, alarmer.Raise(b), "raise failed")
 510         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 511
 512         // Clear two alarms. The first should be delayed. Check the alarms are removed
 513         assert.Nil(t, alarmer.Clear(a), "clear failed")
 514         assert.Nil(t, alarmer.Clear(b), "clear failed")
 515
 516         time.Sleep(time.Duration(2) * time.Second)
 517         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 518         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 519 }
 520
 521 func TestClearExpiredAlarms(t *testing.T) {
 522         xapp.Logger.Info("TestClearExpiredAlarms")
 523
 524         a := alarm.AlarmMessage{
 525                 Alarm:       alarmer.NewAlarm(8007, alarm.SeverityWarning, "threshold", ""),
 526                 AlarmAction: alarm.AlarmActionRaise,
 527                 AlarmTime:   time.Now().UnixNano(),
 528         }
 529         d := alarm.RICAlarmDefinitions[8007]
 530         n := AlarmNotification{a, *d}
 531         alarmManager.activeAlarms = make([]AlarmNotification, 0)
 532         alarmManager.UpdateActiveAlarmList(&n)
 533
 534         // Unknown SP
 535         a.Alarm.SpecificProblem = 1234
 536         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 537
 538         // TTL is 0
 539         d.TimeToLive = 0
 540         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 541
 542         // TTL not expired
 543         a.Alarm.SpecificProblem = 8007
 544         d.TimeToLive = 2
 545         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 546
 547         // TTL expired, alarm should be cleared
 548         time.Sleep(time.Duration(3) * time.Second)
 549         assert.Equal(t, len(alarmManager.activeAlarms), 1)
 550         assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 551         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 552 }
 553
 554 func TestSetAlarmConfig(t *testing.T) {
 555         xapp.Logger.Info("TestSetAlarmConfig")
 556
 557         var setAlarmConfig alarm.AlarmConfigParams
 558         setAlarmConfig.MaxActiveAlarms = 500
 559         setAlarmConfig.MaxAlarmHistory = 2000
 560
 561         pbodyEn, _ := json.Marshal(setAlarmConfig)
 562         req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 563         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 564         response := executeRequest(req, handleFunc)
 565         checkResponseCode(t, http.StatusOK, response.Code)
 566
 567         var getAlarmConfig alarm.AlarmConfigParams
 568         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 569         req = mux.SetURLVars(req, nil)
 570         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 571         response = executeRequest(req, handleFunc)
 572         checkResponseCode(t, http.StatusOK, response.Code)
 573
 574         // Decode the json output from handler
 575         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 576         if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 {
 577                 t.Errorf("Incorrect alarm thresholds")
 578         }
 579
 580         // Revert ot default
 581         setAlarmConfig.MaxActiveAlarms = 5000
 582         setAlarmConfig.MaxAlarmHistory = 20000
 583
 584         pbodyEn, _ = json.Marshal(setAlarmConfig)
 585         req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 586         handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig)
 587         response = executeRequest(req, handleFunc)
 588         checkResponseCode(t, http.StatusOK, response.Code)
 589
 590         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 591         req = mux.SetURLVars(req, nil)
 592         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 593         response = executeRequest(req, handleFunc)
 594         checkResponseCode(t, http.StatusOK, response.Code)
 595
 596         // Decode the json output from handler
 597         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 598         if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 {
 599                 t.Errorf("Incorrect alarm thresholds")
 600         }
 601 }
 602
 603 func TestConfigChangeCB(t *testing.T) {
 604         xapp.Logger.Info("TestConfigChangeCB")
 605         alarmManager.ConfigChangeCB("AlarmManager")
 606 }
 607
 608 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 609         receivedAlert := waitForEvent()
 610
 611         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 612         _, ok := alarmManager.IsMatchFound(a)
 613         assert.True(t, ok)
 614
 615         return receivedAlert
 616 }
 617
 618 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 619         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 620         //assert.Equal(t, receivedAlert, e)
 621 }
 622
 623 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 624         l, err := net.Listen("tcp", "localhost:9093")
 625         if err != nil {
 626                 t.Error("Failed to create listener: " + err.Error())
 627         }
 628         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 629                 assert.Equal(t, r.Method, method)
 630                 assert.Equal(t, r.URL.String(), url)
 631
 632                 fireEvent(t, r.Body)
 633
 634                 w.Header().Add("Content-Type", "application/json")
 635                 w.WriteHeader(status)
 636                 b, _ := json.Marshal(respData)
 637                 w.Write(b)
 638         }))
 639         ts.Listener.Close()
 640         ts.Listener = l
 641
 642         ts.Start()
 643
 644         return ts
 645 }
 646
 647 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
 648         l, err := net.Listen("tcp", "localhost:9093")
 649         if err != nil {
 650                 t.Error("Failed to create listener: " + err.Error())
 651         }
 652         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 653                 assert.Equal(t, r.Method, method)
 654                 assert.Equal(t, r.URL.String(), url)
 655
 656                 w.Header().Add("Content-Type", "application/json")
 657                 w.WriteHeader(200)
 658                 // Read alerts from file
 659                 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
 660                 if err != nil {
 661                         t.Error("Failed to send response: ", err)
 662                 }
 663                 _, err = w.Write(payload)
 664                 if err != nil {
 665                         t.Error("Failed to send response: " + err.Error())
 666                 }
 667         }))
 668         ts.Listener.Close()
 669         ts.Listener = l
 670         ts.Start()
 671         return ts
 672 }
 673
 674 func waitForEvent() string {
 675         receivedAlert := <-eventChan
 676         return receivedAlert
 677 }
 678
 679 func fireEvent(t *testing.T, body io.ReadCloser) {
 680         reqBody, err := ioutil.ReadAll(body)
 681         assert.Nil(t, err, "ioutil.ReadAll failed")
 682         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 683
 684         eventChan <- fmt.Sprintf("%s", reqBody)
 685 }
 686
 687 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 688         rr := httptest.NewRecorder()
 689
 690         handleR.ServeHTTP(rr, req)
 691
 692         return rr
 693 }
 694
 695 func checkResponseCode(t *testing.T, expected, actual int) bool {
 696         if expected != actual {
 697                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 698                 return false
 699         }
 700         return true
 701 }
 702
 703 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
 704         go func() {
 705                 xapp.Logger.Info("Giving CLI command")
 706                 cmd := exec.Command(command, args...)
 707                 cmd.Dir = "../"
 708                 output, err := cmd.CombinedOutput()
 709                 if err != nil {
 710                         xapp.Logger.Info("CLI command failed out: %s", err)
 711                 }
 712                 xapp.Logger.Info("CLI command output: %s", output)
 713                 commandReady <- true
 714                 xapp.Logger.Info("CLI command completed")
 715         }()
 716 }
 717
 718 func readJSONFromFile(filename string) ([]byte, error) {
 719         file, err := ioutil.ReadFile(filename)
 720         if err != nil {
 721                 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
 722                 return nil, err
 723         }
 724         return file, nil
 725 }
 726