manager/cmd/manager_test.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io"
  28         "io/ioutil"
  29         "net"
  30         "net/http"
  31         "net/http/httptest"
  32         "os"
  33         "os/exec"
  34         "strconv"
  35         "strings"
  36         "testing"
  37         "time"
  38
  39         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  40         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  41         "github.com/gorilla/mux"
  42         "github.com/prometheus/alertmanager/api/v2/models"
  43         "github.com/stretchr/testify/assert"
  44 )
  45
  46 var alarmManager *AlarmManager
  47 var alarmer *alarm.RICAlarm
  48 var eventChan chan string
  49
  50 // Test cases
  51 func TestMain(M *testing.M) {
  52         alarmManager = NewAlarmManager("localhost:9093", 500, false)
  53         alarmManager.alertInterval = 20000
  54         go alarmManager.Run(false, 5)
  55         time.Sleep(time.Duration(10) * time.Second)
  56
  57         // Wait until RMR is up-and-running
  58         for !xapp.Rmr.IsReady() {
  59                 time.Sleep(time.Duration(1) * time.Second)
  60         }
  61
  62         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
  63         alarmManager.alarmClient = alarmer
  64         time.Sleep(time.Duration(5) * time.Second)
  65         eventChan = make(chan string)
  66
  67         os.Exit(M.Run())
  68 }
  69
  70 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
  71         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
  72         var alarmDefinition alarm.AlarmDefinition
  73         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
  74         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
  75         req = mux.SetURLVars(req, vars)
  76         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
  77         response := executeRequest(req, handleFunc)
  78         checkResponseCode(t, http.StatusOK, response.Code)
  79         json.NewDecoder(response.Body).Decode(&alarmDefinition)
  80         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
  81         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
  82                 t.Errorf("Incorrect alarm definition")
  83         }
  84 }
  85
  86 func TestSetAlarmDefinitions(t *testing.T) {
  87         xapp.Logger.Info("TestSetAlarmDefinitions")
  88         var alarm8004Definition alarm.AlarmDefinition
  89         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
  90         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
  91         alarm8004Definition.EventType = "Processing error"
  92         alarm8004Definition.OperationInstructions = "Not defined"
  93         alarm8004Definition.RaiseDelay = 0
  94         alarm8004Definition.ClearDelay = 0
  95
  96         var alarm8005Definition alarm.AlarmDefinition
  97         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
  98         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
  99         alarm8005Definition.EventType = "Communication error"
 100         alarm8005Definition.OperationInstructions = "Not defined"
 101         alarm8005Definition.RaiseDelay = 0
 102         alarm8005Definition.ClearDelay = 0
 103
 104         var alarm8006Definition alarm.AlarmDefinition
 105         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
 106         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 107         alarm8006Definition.EventType = "Communication error"
 108         alarm8006Definition.OperationInstructions = "Not defined"
 109         alarm8006Definition.RaiseDelay = 0
 110         alarm8006Definition.ClearDelay = 0
 111
 112         var alarm8007Definition alarm.AlarmDefinition
 113         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 114         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 115         alarm8007Definition.EventType = "Communication error"
 116         alarm8007Definition.OperationInstructions = "Not defined"
 117         alarm8007Definition.RaiseDelay = 0
 118         alarm8007Definition.ClearDelay = 0
 119
 120         var alarm8008Definition alarm.AlarmDefinition
 121         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 122         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 123         alarm8008Definition.EventType = "storage warning"
 124         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
 125         alarm8008Definition.RaiseDelay = 0
 126         alarm8008Definition.ClearDelay = 0
 127
 128         var alarm8009Definition alarm.AlarmDefinition
 129         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 130         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 131         alarm8009Definition.EventType = "storage warning"
 132         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
 133         alarm8009Definition.RaiseDelay = 0
 134         alarm8009Definition.ClearDelay = 0
 135
 136         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 137         pbodyEn, _ := json.Marshal(pbodyParams)
 138         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 139         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 140         response := executeRequest(req, handleFunc)
 141         status := checkResponseCode(t, http.StatusOK, response.Code)
 142         xapp.Logger.Info("status = %v", status)
 143 }
 144
 145
 146 func TestSetAlarmConfigDecodeError(t *testing.T) {
 147         xapp.Logger.Info("TestSetAlarmConfigDecodeError")
 148
 149         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 150         req, _ := http.NewRequest("POST", "/ric/v1/alarms", bytes.NewBuffer(jsonStr))
 151         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 152         response := executeRequest(req, handleFunc)
 153         status := checkResponseCode(t,http.StatusOK,response.Code)
 154         xapp.Logger.Info("status = %v", status)
 155 }
 156
 157 func TestSetAlarmDefinitionDecodeError(t *testing.T) {
 158         xapp.Logger.Info("TestSetAlarmDefinitionDecodeError")
 159
 160         var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
 161         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(jsonStr))
 162         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 163         response := executeRequest(req, handleFunc)
 164         status := checkResponseCode(t,http.StatusBadRequest,response.Code)
 165         xapp.Logger.Info("status = %v", status)
 166 }
 167
 168 func TestRaiseAlarmEmptyBody(t *testing.T) {
 169         xapp.Logger.Info("TestRaiseAlarmEmptyBody")
 170         req, _ := http.NewRequest("POST", "/ric/v1/alarms", nil)
 171         handleFunc := http.HandlerFunc(alarmManager.RaiseAlarm)
 172         response := executeRequest(req, handleFunc)
 173         status := checkResponseCode(t,http.StatusOK,response.Code)
 174         xapp.Logger.Info("status = %v", status)
 175 }
 176
 177 func TestSetAlarmDefinitionsEmptyBody(t *testing.T) {
 178         xapp.Logger.Info("TestSetAlarmDefinitionsEmptyBody")
 179         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", nil)
 180         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 181         response := executeRequest(req, handleFunc)
 182         status := checkResponseCode(t,http.StatusBadRequest,response.Code)
 183         xapp.Logger.Info("status = %v", status)
 184 }
 185
 186 func TestClearAlarmEmptyBody(t *testing.T) {
 187         xapp.Logger.Info("TestClearAlarmEmptyBody")
 188         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms", nil)
 189         handleFunc := http.HandlerFunc(alarmManager.ClearAlarm)
 190         response := executeRequest(req, handleFunc)
 191         status := checkResponseCode(t,http.StatusOK,response.Code)
 192         xapp.Logger.Info("status = %v", status)
 193 }
 194
 195 func TestGetAlarmDefinitions(t *testing.T) {
 196         xapp.Logger.Info("TestGetAlarmDefinitions")
 197         var alarmDefinition alarm.AlarmDefinition
 198         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 199         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 200         req = mux.SetURLVars(req, vars)
 201         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 202         response := executeRequest(req, handleFunc)
 203         checkResponseCode(t, http.StatusOK, response.Code)
 204         json.NewDecoder(response.Body).Decode(&alarmDefinition)
 205         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
 206         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
 207                 t.Errorf("Incorrect alarm definition")
 208         }
 209 }
 210
 211 func TestDeleteAlarmDefinitions(t *testing.T) {
 212         xapp.Logger.Info("TestDeleteAlarmDefinitions")
 213         //Get all
 214         var ricAlarmDefinitions RicAlarmDefinitions
 215         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 216         req = mux.SetURLVars(req, nil)
 217         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
 218         response := executeRequest(req, handleFunc)
 219         checkResponseCode(t, http.StatusOK, response.Code)
 220         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
 221         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
 222                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
 223         }
 224
 225         //Delete 8004
 226         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 227         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 228         req = mux.SetURLVars(req, vars)
 229         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 230         response = executeRequest(req, handleFunc)
 231         checkResponseCode(t, http.StatusOK, response.Code)
 232
 233         //Get 8004 fail
 234         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 235         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 236         req = mux.SetURLVars(req, vars)
 237         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 238         response = executeRequest(req, handleFunc)
 239         checkResponseCode(t, http.StatusBadRequest, response.Code)
 240
 241         //Set 8004 success
 242         var alarm8004Definition alarm.AlarmDefinition
 243         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
 244         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 245         alarm8004Definition.EventType = "Processing error"
 246         alarm8004Definition.OperationInstructions = "Not defined"
 247         alarm8004Definition.RaiseDelay = 0
 248         alarm8004Definition.ClearDelay = 0
 249         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 250         pbodyEn, _ := json.Marshal(pbodyParams)
 251         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 252         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 253         response = executeRequest(req, handleFunc)
 254         checkResponseCode(t, http.StatusOK, response.Code)
 255
 256         //Get 8004 success
 257         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 258         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 259         req = mux.SetURLVars(req, vars)
 260         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 261         response = executeRequest(req, handleFunc)
 262         checkResponseCode(t, http.StatusOK, response.Code)
 263 }
 264
 265 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 266         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
 267         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 268         defer ts.Close()
 269
 270         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 271         assert.Nil(t, alarmer.Raise(a), "raise failed")
 272
 273         VerifyAlarm(t, a, 1)
 274
 275         var activeAlarms []AlarmNotification
 276         activeAlarms = make([]AlarmNotification, 1)
 277         req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil)
 278         req = mux.SetURLVars(req, nil)
 279         handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms)
 280         response := executeRequest(req, handleFunc)
 281         checkResponseCode(t, http.StatusOK, response.Code)
 282
 283         // Decode the json output from handler
 284         json.NewDecoder(response.Body).Decode(activeAlarms)
 285         if len(activeAlarms) != 1 {
 286                 t.Errorf("Incorrect alarm alarm count")
 287         }
 288
 289         var alarmHistory []AlarmNotification
 290         alarmHistory = make([]AlarmNotification, 1)
 291         req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil)
 292         req = mux.SetURLVars(req, nil)
 293         handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory)
 294         response = executeRequest(req, handleFunc)
 295         checkResponseCode(t, http.StatusOK, response.Code)
 296
 297         // Decode the json output from handler
 298         json.NewDecoder(response.Body).Decode(alarmHistory)
 299         if len(alarmHistory) != 1 {
 300                 t.Errorf("Incorrect alarm history count")
 301         }
 302 }
 303
 304 func TestAlarmClearedSucess(t *testing.T) {
 305         xapp.Logger.Info("TestAlarmClearedSucess")
 306         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 307         defer ts.Close()
 308
 309         // Raise the alarm
 310         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 311         assert.Nil(t, alarmer.Raise(a), "raise failed")
 312
 313         VerifyAlarm(t, a, 1)
 314
 315         // Now Clear the alarm and check alarm is removed
 316         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
 317         assert.Nil(t, alarmer.Clear(a), "clear failed")
 318
 319         time.Sleep(time.Duration(2) * time.Second)
 320         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 321 }
 322
 323 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 324         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
 325         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 326         defer ts.Close()
 327
 328         // Raise two alarms
 329         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 330         assert.Nil(t, alarmer.Raise(a), "raise failed")
 331
 332         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 333         assert.Nil(t, alarmer.Raise(b), "raise failed")
 334
 335         time.Sleep(time.Duration(5) * time.Second)
 336
 337         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 338         VerifyAlarm(t, a, 1)
 339         xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
 340         VerifyAlarm(t, b, 2)
 341 }
 342
 343 func TestMultipleAlarmsClearedSucess(t *testing.T) {
 344         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
 345         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 346         defer ts.Close()
 347
 348         // Raise two alarms
 349         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 350         assert.Nil(t, alarmer.Clear(a), "clear failed")
 351
 352         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 353         assert.Nil(t, alarmer.Clear(b), "clear failed")
 354
 355         time.Sleep(time.Duration(2) * time.Second)
 356         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 357 }
 358
 359 func TestAlarmsSuppresedSucess(t *testing.T) {
 360         xapp.Logger.Info("TestAlarmsSuppresedSucess")
 361         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 362         defer ts.Close()
 363
 364         // Raise two similar/matching alarms ... the second one suppresed
 365         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 366         assert.Nil(t, alarmer.Raise(a), "raise failed")
 367         assert.Nil(t, alarmer.Raise(a), "raise failed")
 368
 369         VerifyAlarm(t, a, 1)
 370         assert.Nil(t, alarmer.Clear(a), "clear failed")
 371 }
 372
 373 func TestInvalidAlarms(t *testing.T) {
 374         xapp.Logger.Info("TestInvalidAlarms")
 375         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
 376         assert.Nil(t, alarmer.Raise(a), "raise failed")
 377         time.Sleep(time.Duration(2) * time.Second)
 378 }
 379
 380 func TestAlarmHandlingErrorCases(t *testing.T) {
 381         xapp.Logger.Info("TestAlarmHandlingErrorCases")
 382         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
 383         assert.Equal(t, err.Error(), "unexpected end of JSON input")
 384         assert.Nil(t, ok, "raise failed")
 385 }
 386
 387 func TestConsumeUnknownMessage(t *testing.T) {
 388         xapp.Logger.Info("TestConsumeUnknownMessage")
 389         err := alarmManager.Consume(&xapp.RMRParams{})
 390         assert.Nil(t, err, "raise failed")
 391 }
 392
 393 func TestStatusCallback(t *testing.T) {
 394         xapp.Logger.Info("TestStatusCallback")
 395         assert.Equal(t, true, alarmManager.StatusCB())
 396 }
 397
 398 func TestActiveAlarmMaxThresholds(t *testing.T) {
 399         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
 400         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 401         alarmManager.maxActiveAlarms = 0
 402         alarmManager.maxAlarmHistory = 10
 403
 404         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
 405         assert.Nil(t, alarmer.Raise(a), "raise failed")
 406
 407         var alarmConfigParams alarm.AlarmConfigParams
 408         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 409         req = mux.SetURLVars(req, nil)
 410         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
 411         response := executeRequest(req, handleFunc)
 412
 413         // Check HTTP Status Code
 414         checkResponseCode(t, http.StatusOK, response.Code)
 415
 416         // Decode the json output from handler
 417         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
 418         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
 419                 t.Errorf("Incorrect alarm thresholds")
 420         }
 421
 422         time.Sleep(time.Duration(1) * time.Second)
 423         alarmManager.maxActiveAlarms = 5000
 424         alarmManager.maxAlarmHistory = 20000
 425         VerifyAlarm(t, a, 2)
 426         VerifyAlarm(t, a, 2)
 427         ts.Close()
 428 }
 429
 430 func TestGetPrometheusAlerts(t *testing.T) {
 431         time.Sleep(1 * time.Second)
 432         xapp.Logger.Info("TestGetPrometheusAlerts")
 433         ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
 434
 435         commandReady := make(chan bool, 1)
 436         command := "cli/alarm-cli"
 437         args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
 438         ExecCLICommand(commandReady, command, args...)
 439         <-commandReady
 440
 441         ts.Close()
 442 }
 443
 444 func TestDelayedAlarmRaiseAndClear(t *testing.T) {
 445         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
 446
 447         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 448         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 449
 450         // Add new alarm definition
 451         var alarm9999Definition alarm.AlarmDefinition
 452         alarm9999Definition.AlarmId = 9999
 453         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 454         alarm9999Definition.EventType = "Test type"
 455         alarm9999Definition.OperationInstructions = "Not defined"
 456         alarm9999Definition.RaiseDelay = 1
 457         alarm9999Definition.ClearDelay = 1
 458         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 459         pbodyEn, _ := json.Marshal(pbodyParams)
 460         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 461         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
 462         response := executeRequest(req, handleFunc)
 463         checkResponseCode(t, http.StatusOK, response.Code)
 464
 465         // Verify 9999 alarm definition
 466         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 467         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 468         req = mux.SetURLVars(req, vars)
 469         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 470         response = executeRequest(req, handleFunc)
 471         checkResponseCode(t, http.StatusOK, response.Code)
 472
 473         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 474         defer ts.Close()
 475
 476         // Raise alarm. Posting alert and updating alarm history should be delayed
 477         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 478         assert.Nil(t, alarmer.Raise(a), "raise failed")
 479         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 480
 481         // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
 482         assert.Nil(t, alarmer.Clear(a), "clear failed")
 483
 484         time.Sleep(time.Duration(2) * time.Second)
 485         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 486         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
 487 }
 488
 489 func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
 490         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
 491
 492         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 493         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 494
 495         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 496         defer ts.Close()
 497
 498         // Raise two alarms. The first should be delayed
 499         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 500         assert.Nil(t, alarmer.Raise(a), "raise failed")
 501         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 502
 503         b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 504         assert.Nil(t, alarmer.Raise(b), "raise failed")
 505         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 506
 507         // Clear two alarms. The first should be delayed. Check the alarms are removed
 508         assert.Nil(t, alarmer.Clear(a), "clear failed")
 509         assert.Nil(t, alarmer.Clear(b), "clear failed")
 510
 511         time.Sleep(time.Duration(2) * time.Second)
 512         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 513         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 514 }
 515
 516 func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
 517         xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
 518
 519         // Delete exisitng 9999 alarm definition
 520         req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
 521         vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
 522         req = mux.SetURLVars(req, vars)
 523         handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
 524         response := executeRequest(req, handleFunc)
 525         checkResponseCode(t, http.StatusOK, response.Code)
 526
 527         // Add updated 9999 alarm definition
 528         var alarm9999Definition alarm.AlarmDefinition
 529         alarm9999Definition.AlarmId = 9999
 530         alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
 531         alarm9999Definition.EventType = "Test type"
 532         alarm9999Definition.OperationInstructions = "Not defined"
 533         alarm9999Definition.RaiseDelay = 1
 534         alarm9999Definition.ClearDelay = 0
 535         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
 536         pbodyEn, _ := json.Marshal(pbodyParams)
 537         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
 538         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
 539         response = executeRequest(req, handleFunc)
 540         checkResponseCode(t, http.StatusOK, response.Code)
 541
 542         // Verify 9999 alarm definition
 543         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
 544         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
 545         req = mux.SetURLVars(req, vars)
 546         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
 547         response = executeRequest(req, handleFunc)
 548         checkResponseCode(t, http.StatusOK, response.Code)
 549
 550         activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
 551         alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
 552
 553         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
 554         defer ts.Close()
 555
 556         // Raise two alarms. The first should be delayed
 557         a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
 558         assert.Nil(t, alarmer.Raise(a), "raise failed")
 559         VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
 560
 561         b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
 562         assert.Nil(t, alarmer.Raise(b), "raise failed")
 563         VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
 564
 565         // Clear two alarms. The first should be delayed. Check the alarms are removed
 566         assert.Nil(t, alarmer.Clear(a), "clear failed")
 567         assert.Nil(t, alarmer.Clear(b), "clear failed")
 568
 569         time.Sleep(time.Duration(2) * time.Second)
 570         assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
 571         assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
 572 }
 573
 574 func TestClearExpiredAlarms(t *testing.T) {
 575         xapp.Logger.Info("TestClearExpiredAlarms")
 576
 577         a := alarm.AlarmMessage{
 578                 Alarm:       alarmer.NewAlarm(8007, alarm.SeverityWarning, "threshold", ""),
 579                 AlarmAction: alarm.AlarmActionRaise,
 580                 AlarmTime:   time.Now().UnixNano(),
 581         }
 582         d := alarm.RICAlarmDefinitions[8007]
 583         n := AlarmNotification{a, *d}
 584         alarmManager.activeAlarms = make([]AlarmNotification, 0)
 585         alarmManager.UpdateActiveAlarmList(&n)
 586
 587         // Unknown SP
 588         a.Alarm.SpecificProblem = 1234
 589         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 590
 591         // TTL is 0
 592         d.TimeToLive = 0
 593         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 594
 595         // TTL not expired
 596         a.Alarm.SpecificProblem = 8007
 597         d.TimeToLive = 2
 598         assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 599
 600         // TTL expired, alarm should be cleared
 601         time.Sleep(time.Duration(3) * time.Second)
 602         assert.Equal(t, len(alarmManager.activeAlarms), 1)
 603         assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
 604         assert.Equal(t, len(alarmManager.activeAlarms), 0)
 605 }
 606
 607 func TestSetAlarmConfig(t *testing.T) {
 608         xapp.Logger.Info("TestSetAlarmConfig")
 609
 610         var setAlarmConfig alarm.AlarmConfigParams
 611         setAlarmConfig.MaxActiveAlarms = 500
 612         setAlarmConfig.MaxAlarmHistory = 2000
 613
 614         pbodyEn, _ := json.Marshal(setAlarmConfig)
 615         req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 616         handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
 617         response := executeRequest(req, handleFunc)
 618         checkResponseCode(t, http.StatusOK, response.Code)
 619
 620         var getAlarmConfig alarm.AlarmConfigParams
 621         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 622         req = mux.SetURLVars(req, nil)
 623         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 624         response = executeRequest(req, handleFunc)
 625         checkResponseCode(t, http.StatusOK, response.Code)
 626
 627         // Decode the json output from handler
 628         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 629         if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 {
 630                 t.Errorf("Incorrect alarm thresholds")
 631         }
 632
 633         // Revert ot default
 634         setAlarmConfig.MaxActiveAlarms = 5000
 635         setAlarmConfig.MaxAlarmHistory = 20000
 636
 637         pbodyEn, _ = json.Marshal(setAlarmConfig)
 638         req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
 639         handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig)
 640         response = executeRequest(req, handleFunc)
 641         checkResponseCode(t, http.StatusOK, response.Code)
 642
 643         req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
 644         req = mux.SetURLVars(req, nil)
 645         handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
 646         response = executeRequest(req, handleFunc)
 647         checkResponseCode(t, http.StatusOK, response.Code)
 648
 649         // Decode the json output from handler
 650         json.NewDecoder(response.Body).Decode(&getAlarmConfig)
 651         if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 {
 652                 t.Errorf("Incorrect alarm thresholds")
 653         }
 654 }
 655
 656 func TestConfigChangeCB(t *testing.T) {
 657         xapp.Logger.Info("TestConfigChangeCB")
 658         alarmManager.ConfigChangeCB("AlarmManager")
 659 }
 660
 661 func TestPersistentStorage(t *testing.T) {
 662     xapp.Logger.Info("TestPersistentStorage")
 663     alarmManager.alarmInfoPvFile = "../../definitions/sample.json"
 664     alarmManager.ReadAlarmInfoFromPersistentVolume()
 665 }
 666
 667 func TestReadAlarmDefinitionFromJsonWrongFilename(t *testing.T) {
 668   // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 669     xapp.Logger.Info("TestReadAlarmDefinitionFromJsonWrongFilename")
 670     os.Setenv("DEF_FILE","test.json")
 671     alarmManager.ReadAlarmDefinitionFromJson()
 672   // correct the filename
 673 }
 674
 675 func TestReadAlarmDefinitionFromJsonInvalidFilename(t *testing.T) {
 676   // use   to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
 677     xapp.Logger.Info("TestReadAlarmDefinitionFromJsonInvalidFilename")
 678     os.Setenv("DEF_FILE","../../definitions/test.json")
 679     alarmManager.ReadAlarmDefinitionFromJson()
 680   // correct the filename
 681 }
 682
 683 func TestPostAlarm(t *testing.T) {
 684         xapp.Logger.Info("TestPostAlarm")
 685         var activeAlarms []AlarmNotification
 686         activeAlarms = make([]AlarmNotification, 1)
 687         alarmManager.PostAlarm(&activeAlarms[0])
 688 }
 689
 690 func TestPostAlarm1(t *testing.T) {
 691         xapp.Logger.Info("TestPostAlarm")
 692         var activeAlarms []AlarmNotification
 693         activeAlarms = make([]AlarmNotification, 2)
 694         alarmManager.PostAlarm(&activeAlarms[0])
 695 }
 696
 697 func TestNewAlarmManagerOther(t *testing.T){
 698     NewAlarmManager("", 0, true)
 699 }
 700
 701 func TestStatusCallbackFailure(t *testing.T) {
 702         xapp.Logger.Info("TestStatusCallbackFailure")
 703         alarmManager.rmrReady = false
 704         assert.Equal(t, false, alarmManager.StatusCB())
 705 }
 706
 707 func TestConfigChangeCBFailure(t *testing.T) {
 708         xapp.Logger.Info("TestConfigChangeCBFailure")
 709         alarmManager.maxActiveAlarms = 0
 710         alarmManager.maxAlarmHistory = 0
 711         alarmManager.ConfigChangeCB("AlarmManager")
 712 }
 713
 714
 715 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 716         receivedAlert := waitForEvent()
 717
 718         assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 719         _, ok := alarmManager.IsMatchFound(a)
 720         assert.True(t, ok)
 721
 722         return receivedAlert
 723 }
 724
 725 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
 726         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
 727         //assert.Equal(t, receivedAlert, e)
 728 }
 729
 730 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
 731         l, err := net.Listen("tcp", "localhost:9093")
 732         if err != nil {
 733                 t.Error("Failed to create listener: " + err.Error())
 734         }
 735         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 736                 assert.Equal(t, r.Method, method)
 737                 assert.Equal(t, r.URL.String(), url)
 738
 739                 fireEvent(t, r.Body)
 740
 741                 w.Header().Add("Content-Type", "application/json")
 742                 w.WriteHeader(status)
 743                 b, _ := json.Marshal(respData)
 744                 w.Write(b)
 745         }))
 746         ts.Listener.Close()
 747         ts.Listener = l
 748
 749         ts.Start()
 750
 751         return ts
 752 }
 753
 754 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
 755         l, err := net.Listen("tcp", "localhost:9093")
 756         if err != nil {
 757                 t.Error("Failed to create listener: " + err.Error())
 758         }
 759         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 760                 assert.Equal(t, r.Method, method)
 761                 assert.Equal(t, r.URL.String(), url)
 762
 763                 w.Header().Add("Content-Type", "application/json")
 764                 w.WriteHeader(200)
 765                 // Read alerts from file
 766                 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
 767                 if err != nil {
 768                         t.Error("Failed to send response: ", err)
 769                 }
 770                 _, err = w.Write(payload)
 771                 if err != nil {
 772                         t.Error("Failed to send response: " + err.Error())
 773                 }
 774         }))
 775         ts.Listener.Close()
 776         ts.Listener = l
 777         ts.Start()
 778         return ts
 779 }
 780
 781 func waitForEvent() string {
 782         receivedAlert := <-eventChan
 783         return receivedAlert
 784 }
 785
 786 func fireEvent(t *testing.T, body io.ReadCloser) {
 787         reqBody, err := ioutil.ReadAll(body)
 788         assert.Nil(t, err, "ioutil.ReadAll failed")
 789         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
 790
 791         eventChan <- fmt.Sprintf("%s", reqBody)
 792 }
 793
 794 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
 795         rr := httptest.NewRecorder()
 796
 797         handleR.ServeHTTP(rr, req)
 798
 799         return rr
 800 }
 801
 802 func checkResponseCode(t *testing.T, expected, actual int) bool {
 803         if expected != actual {
 804                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
 805                 return false
 806         }
 807         return true
 808 }
 809
 810 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
 811         go func() {
 812                 xapp.Logger.Info("Giving CLI command")
 813                 cmd := exec.Command(command, args...)
 814                 cmd.Dir = "../"
 815                 output, err := cmd.CombinedOutput()
 816                 if err != nil {
 817                         xapp.Logger.Info("CLI command failed out: %s", err)
 818                 }
 819                 xapp.Logger.Info("CLI command output: %s", output)
 820                 commandReady <- true
 821                 xapp.Logger.Info("CLI command completed")
 822         }()
 823 }
 824
 825 func readJSONFromFile(filename string) ([]byte, error) {
 826         file, err := ioutil.ReadFile(filename)
 827         if err != nil {
 828                 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
 829                 return nil, err
 830         }
 831         return file, nil
 832 }
 833
 834