2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
39 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
40 "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
41 "github.com/gorilla/mux"
42 "github.com/prometheus/alertmanager/api/v2/models"
43 "github.com/stretchr/testify/assert"
46 var alarmManager *AlarmManager
47 var alarmer *alarm.RICAlarm
48 var eventChan chan string
51 func TestMain(M *testing.M) {
52 alarmManager = NewAlarmManager("localhost:9093", 500, false)
53 alarmManager.alertInterval = 20000
54 go alarmManager.Run(false, 5)
55 time.Sleep(time.Duration(10) * time.Second)
57 // Wait until RMR is up-and-running
58 for !xapp.Rmr.IsReady() {
59 time.Sleep(time.Duration(1) * time.Second)
62 alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
63 alarmManager.alarmClient = alarmer
64 time.Sleep(time.Duration(5) * time.Second)
65 eventChan = make(chan string)
70 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
71 xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
72 var alarmDefinition alarm.AlarmDefinition
73 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
74 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
75 req = mux.SetURLVars(req, vars)
76 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
77 response := executeRequest(req, handleFunc)
78 checkResponseCode(t, http.StatusOK, response.Code)
79 json.NewDecoder(response.Body).Decode(&alarmDefinition)
80 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
81 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
82 t.Errorf("Incorrect alarm definition")
86 func TestSetAlarmDefinitions(t *testing.T) {
87 xapp.Logger.Info("TestSetAlarmDefinitions")
88 var alarm8004Definition alarm.AlarmDefinition
89 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
90 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
91 alarm8004Definition.EventType = "Processing error"
92 alarm8004Definition.OperationInstructions = "Not defined"
93 alarm8004Definition.RaiseDelay = 0
94 alarm8004Definition.ClearDelay = 0
96 var alarm8005Definition alarm.AlarmDefinition
97 alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
98 alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
99 alarm8005Definition.EventType = "Communication error"
100 alarm8005Definition.OperationInstructions = "Not defined"
101 alarm8005Definition.RaiseDelay = 0
102 alarm8005Definition.ClearDelay = 0
104 var alarm8006Definition alarm.AlarmDefinition
105 alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
106 alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
107 alarm8006Definition.EventType = "Communication error"
108 alarm8006Definition.OperationInstructions = "Not defined"
109 alarm8006Definition.RaiseDelay = 0
110 alarm8006Definition.ClearDelay = 0
112 var alarm8007Definition alarm.AlarmDefinition
113 alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
114 alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
115 alarm8007Definition.EventType = "Communication error"
116 alarm8007Definition.OperationInstructions = "Not defined"
117 alarm8007Definition.RaiseDelay = 0
118 alarm8007Definition.ClearDelay = 0
120 var alarm8008Definition alarm.AlarmDefinition
121 alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
122 alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
123 alarm8008Definition.EventType = "storage warning"
124 alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
125 alarm8008Definition.RaiseDelay = 0
126 alarm8008Definition.ClearDelay = 0
128 var alarm8009Definition alarm.AlarmDefinition
129 alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
130 alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
131 alarm8009Definition.EventType = "storage warning"
132 alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
133 alarm8009Definition.RaiseDelay = 0
134 alarm8009Definition.ClearDelay = 0
136 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
137 pbodyEn, _ := json.Marshal(pbodyParams)
138 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
139 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
140 response := executeRequest(req, handleFunc)
141 status := checkResponseCode(t, http.StatusOK, response.Code)
142 xapp.Logger.Info("status = %v", status)
146 func TestSetAlarmConfigDecodeError(t *testing.T) {
147 xapp.Logger.Info("TestSetAlarmConfigDecodeError")
149 var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
150 req, _ := http.NewRequest("POST", "/ric/v1/alarms", bytes.NewBuffer(jsonStr))
151 handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
152 response := executeRequest(req, handleFunc)
153 status := checkResponseCode(t,http.StatusOK,response.Code)
154 xapp.Logger.Info("status = %v", status)
157 func TestSetAlarmDefinitionDecodeError(t *testing.T) {
158 xapp.Logger.Info("TestSetAlarmDefinitionDecodeError")
160 var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`)
161 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(jsonStr))
162 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
163 response := executeRequest(req, handleFunc)
164 status := checkResponseCode(t,http.StatusBadRequest,response.Code)
165 xapp.Logger.Info("status = %v", status)
168 func TestRaiseAlarmEmptyBody(t *testing.T) {
169 xapp.Logger.Info("TestRaiseAlarmEmptyBody")
170 req, _ := http.NewRequest("POST", "/ric/v1/alarms", nil)
171 handleFunc := http.HandlerFunc(alarmManager.RaiseAlarm)
172 response := executeRequest(req, handleFunc)
173 status := checkResponseCode(t,http.StatusOK,response.Code)
174 xapp.Logger.Info("status = %v", status)
177 func TestSetAlarmDefinitionsEmptyBody(t *testing.T) {
178 xapp.Logger.Info("TestSetAlarmDefinitionsEmptyBody")
179 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", nil)
180 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
181 response := executeRequest(req, handleFunc)
182 status := checkResponseCode(t,http.StatusBadRequest,response.Code)
183 xapp.Logger.Info("status = %v", status)
186 func TestClearAlarmEmptyBody(t *testing.T) {
187 xapp.Logger.Info("TestClearAlarmEmptyBody")
188 req, _ := http.NewRequest("DELETE", "/ric/v1/alarms", nil)
189 handleFunc := http.HandlerFunc(alarmManager.ClearAlarm)
190 response := executeRequest(req, handleFunc)
191 status := checkResponseCode(t,http.StatusOK,response.Code)
192 xapp.Logger.Info("status = %v", status)
195 func TestGetAlarmDefinitions(t *testing.T) {
196 xapp.Logger.Info("TestGetAlarmDefinitions")
197 var alarmDefinition alarm.AlarmDefinition
198 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
199 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
200 req = mux.SetURLVars(req, vars)
201 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
202 response := executeRequest(req, handleFunc)
203 checkResponseCode(t, http.StatusOK, response.Code)
204 json.NewDecoder(response.Body).Decode(&alarmDefinition)
205 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
206 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
207 t.Errorf("Incorrect alarm definition")
211 func TestDeleteAlarmDefinitions(t *testing.T) {
212 xapp.Logger.Info("TestDeleteAlarmDefinitions")
214 var ricAlarmDefinitions RicAlarmDefinitions
215 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
216 req = mux.SetURLVars(req, nil)
217 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
218 response := executeRequest(req, handleFunc)
219 checkResponseCode(t, http.StatusOK, response.Code)
220 json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
221 for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
222 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
226 req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
227 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
228 req = mux.SetURLVars(req, vars)
229 handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
230 response = executeRequest(req, handleFunc)
231 checkResponseCode(t, http.StatusOK, response.Code)
234 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
235 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
236 req = mux.SetURLVars(req, vars)
237 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
238 response = executeRequest(req, handleFunc)
239 checkResponseCode(t, http.StatusBadRequest, response.Code)
242 var alarm8004Definition alarm.AlarmDefinition
243 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
244 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
245 alarm8004Definition.EventType = "Processing error"
246 alarm8004Definition.OperationInstructions = "Not defined"
247 alarm8004Definition.RaiseDelay = 0
248 alarm8004Definition.ClearDelay = 0
249 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
250 pbodyEn, _ := json.Marshal(pbodyParams)
251 req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
252 handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
253 response = executeRequest(req, handleFunc)
254 checkResponseCode(t, http.StatusOK, response.Code)
257 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
258 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
259 req = mux.SetURLVars(req, vars)
260 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
261 response = executeRequest(req, handleFunc)
262 checkResponseCode(t, http.StatusOK, response.Code)
265 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
266 xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
267 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
270 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
271 assert.Nil(t, alarmer.Raise(a), "raise failed")
275 var activeAlarms []AlarmNotification
276 activeAlarms = make([]AlarmNotification, 1)
277 req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil)
278 req = mux.SetURLVars(req, nil)
279 handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms)
280 response := executeRequest(req, handleFunc)
281 checkResponseCode(t, http.StatusOK, response.Code)
283 // Decode the json output from handler
284 json.NewDecoder(response.Body).Decode(activeAlarms)
285 if len(activeAlarms) != 1 {
286 t.Errorf("Incorrect alarm alarm count")
289 var alarmHistory []AlarmNotification
290 alarmHistory = make([]AlarmNotification, 1)
291 req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil)
292 req = mux.SetURLVars(req, nil)
293 handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory)
294 response = executeRequest(req, handleFunc)
295 checkResponseCode(t, http.StatusOK, response.Code)
297 // Decode the json output from handler
298 json.NewDecoder(response.Body).Decode(alarmHistory)
299 if len(alarmHistory) != 1 {
300 t.Errorf("Incorrect alarm history count")
304 func TestAlarmClearedSucess(t *testing.T) {
305 xapp.Logger.Info("TestAlarmClearedSucess")
306 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
310 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
311 assert.Nil(t, alarmer.Raise(a), "raise failed")
315 // Now Clear the alarm and check alarm is removed
316 a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
317 assert.Nil(t, alarmer.Clear(a), "clear failed")
319 time.Sleep(time.Duration(2) * time.Second)
320 assert.Equal(t, len(alarmManager.activeAlarms), 0)
323 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
324 xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
325 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
329 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
330 assert.Nil(t, alarmer.Raise(a), "raise failed")
332 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
333 assert.Nil(t, alarmer.Raise(b), "raise failed")
335 time.Sleep(time.Duration(5) * time.Second)
337 xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
339 xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms)
343 func TestMultipleAlarmsClearedSucess(t *testing.T) {
344 xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
345 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
349 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
350 assert.Nil(t, alarmer.Clear(a), "clear failed")
352 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
353 assert.Nil(t, alarmer.Clear(b), "clear failed")
355 time.Sleep(time.Duration(2) * time.Second)
356 assert.Equal(t, len(alarmManager.activeAlarms), 0)
359 func TestAlarmsSuppresedSucess(t *testing.T) {
360 xapp.Logger.Info("TestAlarmsSuppresedSucess")
361 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
364 // Raise two similar/matching alarms ... the second one suppresed
365 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
366 assert.Nil(t, alarmer.Raise(a), "raise failed")
367 assert.Nil(t, alarmer.Raise(a), "raise failed")
370 assert.Nil(t, alarmer.Clear(a), "clear failed")
373 func TestInvalidAlarms(t *testing.T) {
374 xapp.Logger.Info("TestInvalidAlarms")
375 a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
376 assert.Nil(t, alarmer.Raise(a), "raise failed")
377 time.Sleep(time.Duration(2) * time.Second)
380 func TestAlarmHandlingErrorCases(t *testing.T) {
381 xapp.Logger.Info("TestAlarmHandlingErrorCases")
382 ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
383 assert.Equal(t, err.Error(), "unexpected end of JSON input")
384 assert.Nil(t, ok, "raise failed")
387 func TestConsumeUnknownMessage(t *testing.T) {
388 xapp.Logger.Info("TestConsumeUnknownMessage")
389 err := alarmManager.Consume(&xapp.RMRParams{})
390 assert.Nil(t, err, "raise failed")
393 func TestStatusCallback(t *testing.T) {
394 xapp.Logger.Info("TestStatusCallback")
395 assert.Equal(t, true, alarmManager.StatusCB())
398 func TestActiveAlarmMaxThresholds(t *testing.T) {
399 xapp.Logger.Info("TestActiveAlarmMaxThresholds")
400 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
401 alarmManager.maxActiveAlarms = 0
402 alarmManager.maxAlarmHistory = 10
404 a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
405 assert.Nil(t, alarmer.Raise(a), "raise failed")
407 var alarmConfigParams alarm.AlarmConfigParams
408 req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
409 req = mux.SetURLVars(req, nil)
410 handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
411 response := executeRequest(req, handleFunc)
413 // Check HTTP Status Code
414 checkResponseCode(t, http.StatusOK, response.Code)
416 // Decode the json output from handler
417 json.NewDecoder(response.Body).Decode(&alarmConfigParams)
418 if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
419 t.Errorf("Incorrect alarm thresholds")
422 time.Sleep(time.Duration(1) * time.Second)
423 alarmManager.maxActiveAlarms = 5000
424 alarmManager.maxAlarmHistory = 20000
430 func TestGetPrometheusAlerts(t *testing.T) {
431 time.Sleep(1 * time.Second)
432 xapp.Logger.Info("TestGetPrometheusAlerts")
433 ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
435 commandReady := make(chan bool, 1)
436 command := "cli/alarm-cli"
437 args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
438 ExecCLICommand(commandReady, command, args...)
444 func TestDelayedAlarmRaiseAndClear(t *testing.T) {
445 xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
447 activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
448 alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
450 // Add new alarm definition
451 var alarm9999Definition alarm.AlarmDefinition
452 alarm9999Definition.AlarmId = 9999
453 alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
454 alarm9999Definition.EventType = "Test type"
455 alarm9999Definition.OperationInstructions = "Not defined"
456 alarm9999Definition.RaiseDelay = 1
457 alarm9999Definition.ClearDelay = 1
458 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
459 pbodyEn, _ := json.Marshal(pbodyParams)
460 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
461 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
462 response := executeRequest(req, handleFunc)
463 checkResponseCode(t, http.StatusOK, response.Code)
465 // Verify 9999 alarm definition
466 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
467 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
468 req = mux.SetURLVars(req, vars)
469 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
470 response = executeRequest(req, handleFunc)
471 checkResponseCode(t, http.StatusOK, response.Code)
473 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
476 // Raise alarm. Posting alert and updating alarm history should be delayed
477 a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
478 assert.Nil(t, alarmer.Raise(a), "raise failed")
479 VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
481 // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
482 assert.Nil(t, alarmer.Clear(a), "clear failed")
484 time.Sleep(time.Duration(2) * time.Second)
485 assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
486 assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
489 func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
490 xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
492 activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
493 alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
495 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
498 // Raise two alarms. The first should be delayed
499 a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
500 assert.Nil(t, alarmer.Raise(a), "raise failed")
501 VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
503 b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
504 assert.Nil(t, alarmer.Raise(b), "raise failed")
505 VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
507 // Clear two alarms. The first should be delayed. Check the alarms are removed
508 assert.Nil(t, alarmer.Clear(a), "clear failed")
509 assert.Nil(t, alarmer.Clear(b), "clear failed")
511 time.Sleep(time.Duration(2) * time.Second)
512 assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
513 assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
516 func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
517 xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
519 // Delete exisitng 9999 alarm definition
520 req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
521 vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
522 req = mux.SetURLVars(req, vars)
523 handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
524 response := executeRequest(req, handleFunc)
525 checkResponseCode(t, http.StatusOK, response.Code)
527 // Add updated 9999 alarm definition
528 var alarm9999Definition alarm.AlarmDefinition
529 alarm9999Definition.AlarmId = 9999
530 alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
531 alarm9999Definition.EventType = "Test type"
532 alarm9999Definition.OperationInstructions = "Not defined"
533 alarm9999Definition.RaiseDelay = 1
534 alarm9999Definition.ClearDelay = 0
535 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
536 pbodyEn, _ := json.Marshal(pbodyParams)
537 req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
538 handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
539 response = executeRequest(req, handleFunc)
540 checkResponseCode(t, http.StatusOK, response.Code)
542 // Verify 9999 alarm definition
543 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
544 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
545 req = mux.SetURLVars(req, vars)
546 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
547 response = executeRequest(req, handleFunc)
548 checkResponseCode(t, http.StatusOK, response.Code)
550 activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
551 alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
553 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
556 // Raise two alarms. The first should be delayed
557 a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
558 assert.Nil(t, alarmer.Raise(a), "raise failed")
559 VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
561 b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
562 assert.Nil(t, alarmer.Raise(b), "raise failed")
563 VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
565 // Clear two alarms. The first should be delayed. Check the alarms are removed
566 assert.Nil(t, alarmer.Clear(a), "clear failed")
567 assert.Nil(t, alarmer.Clear(b), "clear failed")
569 time.Sleep(time.Duration(2) * time.Second)
570 assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
571 assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
574 func TestClearExpiredAlarms(t *testing.T) {
575 xapp.Logger.Info("TestClearExpiredAlarms")
577 a := alarm.AlarmMessage{
578 Alarm: alarmer.NewAlarm(8007, alarm.SeverityWarning, "threshold", ""),
579 AlarmAction: alarm.AlarmActionRaise,
580 AlarmTime: time.Now().UnixNano(),
582 d := alarm.RICAlarmDefinitions[8007]
583 n := AlarmNotification{a, *d}
584 alarmManager.activeAlarms = make([]AlarmNotification, 0)
585 alarmManager.UpdateActiveAlarmList(&n)
588 a.Alarm.SpecificProblem = 1234
589 assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
593 assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
596 a.Alarm.SpecificProblem = 8007
598 assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
600 // TTL expired, alarm should be cleared
601 time.Sleep(time.Duration(3) * time.Second)
602 assert.Equal(t, len(alarmManager.activeAlarms), 1)
603 assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
604 assert.Equal(t, len(alarmManager.activeAlarms), 0)
607 func TestSetAlarmConfig(t *testing.T) {
608 xapp.Logger.Info("TestSetAlarmConfig")
610 var setAlarmConfig alarm.AlarmConfigParams
611 setAlarmConfig.MaxActiveAlarms = 500
612 setAlarmConfig.MaxAlarmHistory = 2000
614 pbodyEn, _ := json.Marshal(setAlarmConfig)
615 req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
616 handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig)
617 response := executeRequest(req, handleFunc)
618 checkResponseCode(t, http.StatusOK, response.Code)
620 var getAlarmConfig alarm.AlarmConfigParams
621 req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
622 req = mux.SetURLVars(req, nil)
623 handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
624 response = executeRequest(req, handleFunc)
625 checkResponseCode(t, http.StatusOK, response.Code)
627 // Decode the json output from handler
628 json.NewDecoder(response.Body).Decode(&getAlarmConfig)
629 if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 {
630 t.Errorf("Incorrect alarm thresholds")
634 setAlarmConfig.MaxActiveAlarms = 5000
635 setAlarmConfig.MaxAlarmHistory = 20000
637 pbodyEn, _ = json.Marshal(setAlarmConfig)
638 req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn))
639 handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig)
640 response = executeRequest(req, handleFunc)
641 checkResponseCode(t, http.StatusOK, response.Code)
643 req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil)
644 req = mux.SetURLVars(req, nil)
645 handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig)
646 response = executeRequest(req, handleFunc)
647 checkResponseCode(t, http.StatusOK, response.Code)
649 // Decode the json output from handler
650 json.NewDecoder(response.Body).Decode(&getAlarmConfig)
651 if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 {
652 t.Errorf("Incorrect alarm thresholds")
656 func TestConfigChangeCB(t *testing.T) {
657 xapp.Logger.Info("TestConfigChangeCB")
658 alarmManager.ConfigChangeCB("AlarmManager")
661 func TestPersistentStorage(t *testing.T) {
662 xapp.Logger.Info("TestPersistentStorage")
663 alarmManager.alarmInfoPvFile = "../../definitions/sample.json"
664 alarmManager.ReadAlarmInfoFromPersistentVolume()
667 func TestReadAlarmDefinitionFromJsonWrongFilename(t *testing.T) {
668 // use to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
669 xapp.Logger.Info("TestReadAlarmDefinitionFromJsonWrongFilename")
670 os.Setenv("DEF_FILE","test.json")
671 alarmManager.ReadAlarmDefinitionFromJson()
672 // correct the filename
675 func TestReadAlarmDefinitionFromJsonInvalidFilename(t *testing.T) {
676 // use to set wrong file name os.Setenv("SITE_TITLE", "Test Site")
677 xapp.Logger.Info("TestReadAlarmDefinitionFromJsonInvalidFilename")
678 os.Setenv("DEF_FILE","../../definitions/test.json")
679 alarmManager.ReadAlarmDefinitionFromJson()
680 // correct the filename
683 func TestPostAlarm(t *testing.T) {
684 xapp.Logger.Info("TestPostAlarm")
685 var activeAlarms []AlarmNotification
686 activeAlarms = make([]AlarmNotification, 1)
687 alarmManager.PostAlarm(&activeAlarms[0])
690 func TestPostAlarm1(t *testing.T) {
691 xapp.Logger.Info("TestPostAlarm")
692 var activeAlarms []AlarmNotification
693 activeAlarms = make([]AlarmNotification, 2)
694 alarmManager.PostAlarm(&activeAlarms[0])
697 func TestNewAlarmManagerOther(t *testing.T){
698 NewAlarmManager("", 0, true)
701 func TestStatusCallbackFailure(t *testing.T) {
702 xapp.Logger.Info("TestStatusCallbackFailure")
703 alarmManager.rmrReady = false
704 assert.Equal(t, false, alarmManager.StatusCB())
707 func TestConfigChangeCBFailure(t *testing.T) {
708 xapp.Logger.Info("TestConfigChangeCBFailure")
709 alarmManager.maxActiveAlarms = 0
710 alarmManager.maxAlarmHistory = 0
711 alarmManager.ConfigChangeCB("AlarmManager")
715 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
716 receivedAlert := waitForEvent()
718 assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
719 _, ok := alarmManager.IsMatchFound(a)
725 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
726 receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
727 //assert.Equal(t, receivedAlert, e)
730 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
731 l, err := net.Listen("tcp", "localhost:9093")
733 t.Error("Failed to create listener: " + err.Error())
735 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
736 assert.Equal(t, r.Method, method)
737 assert.Equal(t, r.URL.String(), url)
741 w.Header().Add("Content-Type", "application/json")
742 w.WriteHeader(status)
743 b, _ := json.Marshal(respData)
754 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
755 l, err := net.Listen("tcp", "localhost:9093")
757 t.Error("Failed to create listener: " + err.Error())
759 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
760 assert.Equal(t, r.Method, method)
761 assert.Equal(t, r.URL.String(), url)
763 w.Header().Add("Content-Type", "application/json")
765 // Read alerts from file
766 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
768 t.Error("Failed to send response: ", err)
770 _, err = w.Write(payload)
772 t.Error("Failed to send response: " + err.Error())
781 func waitForEvent() string {
782 receivedAlert := <-eventChan
786 func fireEvent(t *testing.T, body io.ReadCloser) {
787 reqBody, err := ioutil.ReadAll(body)
788 assert.Nil(t, err, "ioutil.ReadAll failed")
789 assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
791 eventChan <- fmt.Sprintf("%s", reqBody)
794 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
795 rr := httptest.NewRecorder()
797 handleR.ServeHTTP(rr, req)
802 func checkResponseCode(t *testing.T, expected, actual int) bool {
803 if expected != actual {
804 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
810 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
812 xapp.Logger.Info("Giving CLI command")
813 cmd := exec.Command(command, args...)
815 output, err := cmd.CombinedOutput()
817 xapp.Logger.Info("CLI command failed out: %s", err)
819 xapp.Logger.Info("CLI command output: %s", output)
821 xapp.Logger.Info("CLI command completed")
825 func readJSONFromFile(filename string) ([]byte, error) {
826 file, err := ioutil.ReadFile(filename)
828 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)