2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
27 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
28 "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
29 "github.com/gorilla/mux"
30 "github.com/prometheus/alertmanager/api/v2/models"
31 "github.com/stretchr/testify/assert"
45 var alarmManager *AlarmManager
46 var alarmer *alarm.RICAlarm
47 var eventChan chan string
50 func TestMain(M *testing.M) {
51 alarmManager = NewAlarmManager("localhost:9093", 500)
52 alarmManager.alertInterval = 20000
53 go alarmManager.Run(false)
54 time.Sleep(time.Duration(10) * time.Second)
56 // Wait until RMR is up-and-running
57 for !xapp.Rmr.IsReady() {
58 time.Sleep(time.Duration(1) * time.Second)
61 alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
62 alarmManager.alarmClient = alarmer
63 time.Sleep(time.Duration(5) * time.Second)
64 eventChan = make(chan string)
69 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
70 xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
71 var alarmDefinition alarm.AlarmDefinition
72 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
73 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
74 req = mux.SetURLVars(req, vars)
75 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
76 response := executeRequest(req, handleFunc)
77 checkResponseCode(t, http.StatusOK, response.Code)
78 json.NewDecoder(response.Body).Decode(&alarmDefinition)
79 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
80 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
81 t.Errorf("Incorrect alarm definition")
85 func TestSetAlarmDefinitions(t *testing.T) {
86 xapp.Logger.Info("TestSetAlarmDefinitions")
87 var alarm8004Definition alarm.AlarmDefinition
88 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
89 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
90 alarm8004Definition.EventType = "Processing error"
91 alarm8004Definition.OperationInstructions = "Not defined"
93 var alarm8005Definition alarm.AlarmDefinition
94 alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
95 alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
96 alarm8005Definition.EventType = "Communication error"
97 alarm8005Definition.OperationInstructions = "Not defined"
99 var alarm8006Definition alarm.AlarmDefinition
100 alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
101 alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
102 alarm8006Definition.EventType = "Communication error"
103 alarm8006Definition.OperationInstructions = "Not defined"
105 var alarm8007Definition alarm.AlarmDefinition
106 alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
107 alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
108 alarm8007Definition.EventType = "Communication error"
109 alarm8007Definition.OperationInstructions = "Not defined"
111 var alarm8008Definition alarm.AlarmDefinition
112 alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
113 alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
114 alarm8008Definition.EventType = "storage warning"
115 alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
117 var alarm8009Definition alarm.AlarmDefinition
118 alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
119 alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
120 alarm8009Definition.EventType = "storage warning"
121 alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
123 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
124 pbodyEn, _ := json.Marshal(pbodyParams)
125 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
126 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
127 response := executeRequest(req, handleFunc)
128 status := checkResponseCode(t, http.StatusOK, response.Code)
129 xapp.Logger.Info("status = %v", status)
133 func TestGetAlarmDefinitions(t *testing.T) {
134 xapp.Logger.Info("TestGetAlarmDefinitions")
135 var alarmDefinition alarm.AlarmDefinition
136 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
137 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
138 req = mux.SetURLVars(req, vars)
139 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
140 response := executeRequest(req, handleFunc)
141 checkResponseCode(t, http.StatusOK, response.Code)
142 json.NewDecoder(response.Body).Decode(&alarmDefinition)
143 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
144 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
145 t.Errorf("Incorrect alarm definition")
149 func TestDeleteAlarmDefinitions(t *testing.T) {
150 xapp.Logger.Info("TestDeleteAlarmDefinitions")
152 var ricAlarmDefinitions RicAlarmDefinitions
153 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
154 req = mux.SetURLVars(req, nil)
155 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
156 response := executeRequest(req, handleFunc)
157 checkResponseCode(t, http.StatusOK, response.Code)
158 json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
159 for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
160 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
164 req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
165 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
166 req = mux.SetURLVars(req, vars)
167 handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
168 response = executeRequest(req, handleFunc)
169 checkResponseCode(t, http.StatusOK, response.Code)
172 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
173 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
174 req = mux.SetURLVars(req, vars)
175 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
176 response = executeRequest(req, handleFunc)
177 checkResponseCode(t, http.StatusBadRequest, response.Code)
180 var alarm8004Definition alarm.AlarmDefinition
181 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
182 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
183 alarm8004Definition.EventType = "Processing error"
184 alarm8004Definition.OperationInstructions = "Not defined"
185 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
186 pbodyEn, _ := json.Marshal(pbodyParams)
187 req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
188 handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
189 response = executeRequest(req, handleFunc)
190 checkResponseCode(t, http.StatusOK, response.Code)
193 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
194 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
195 req = mux.SetURLVars(req, vars)
196 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
197 response = executeRequest(req, handleFunc)
198 checkResponseCode(t, http.StatusOK, response.Code)
201 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
202 xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
203 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
206 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
207 assert.Nil(t, alarmer.Raise(a), "raise failed")
212 func TestAlarmClearedSucess(t *testing.T) {
213 xapp.Logger.Info("TestAlarmClearedSucess")
214 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
218 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
219 assert.Nil(t, alarmer.Raise(a), "raise failed")
223 // Now Clear the alarm and check alarm is removed
224 a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
225 assert.Nil(t, alarmer.Clear(a), "clear failed")
227 time.Sleep(time.Duration(2) * time.Second)
228 assert.Equal(t, len(alarmManager.activeAlarms), 0)
231 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
232 xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
233 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
237 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
238 assert.Nil(t, alarmer.Raise(a), "raise failed")
240 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
241 assert.Nil(t, alarmer.Raise(b), "raise failed")
243 time.Sleep(time.Duration(2) * time.Second)
248 func TestMultipleAlarmsClearedSucess(t *testing.T) {
249 xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
250 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
254 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
255 assert.Nil(t, alarmer.Clear(a), "clear failed")
257 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
258 assert.Nil(t, alarmer.Clear(b), "clear failed")
260 time.Sleep(time.Duration(2) * time.Second)
261 assert.Equal(t, len(alarmManager.activeAlarms), 0)
264 func TestAlarmsSuppresedSucess(t *testing.T) {
265 xapp.Logger.Info("TestAlarmsSuppresedSucess")
266 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
269 // Raise two similar/matching alarms ... the second one suppresed
270 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
271 assert.Nil(t, alarmer.Raise(a), "raise failed")
272 assert.Nil(t, alarmer.Raise(a), "raise failed")
275 assert.Nil(t, alarmer.Clear(a), "clear failed")
278 func TestInvalidAlarms(t *testing.T) {
279 xapp.Logger.Info("TestInvalidAlarms")
280 a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
281 assert.Nil(t, alarmer.Raise(a), "raise failed")
282 time.Sleep(time.Duration(2) * time.Second)
285 func TestAlarmHandlingErrorCases(t *testing.T) {
286 xapp.Logger.Info("TestAlarmHandlingErrorCases")
287 ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
288 assert.Equal(t, err.Error(), "unexpected end of JSON input")
289 assert.Nil(t, ok, "raise failed")
292 func TestConsumeUnknownMessage(t *testing.T) {
293 xapp.Logger.Info("TestConsumeUnknownMessage")
294 err := alarmManager.Consume(&xapp.RMRParams{})
295 assert.Nil(t, err, "raise failed")
298 func TestStatusCallback(t *testing.T) {
299 xapp.Logger.Info("TestStatusCallback")
300 assert.Equal(t, true, alarmManager.StatusCB())
303 func TestActiveAlarmMaxThresholds(t *testing.T) {
304 xapp.Logger.Info("TestActiveAlarmMaxThresholds")
305 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
306 alarmManager.maxActiveAlarms = 0
307 alarmManager.maxAlarmHistory = 10
309 a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
310 assert.Nil(t, alarmer.Raise(a), "raise failed")
312 var alarmConfigParams alarm.AlarmConfigParams
313 req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
314 req = mux.SetURLVars(req, nil)
315 handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
316 response := executeRequest(req, handleFunc)
318 // Check HTTP Status Code
319 checkResponseCode(t, http.StatusOK, response.Code)
321 // Decode the json output from handler
322 json.NewDecoder(response.Body).Decode(&alarmConfigParams)
323 if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
324 t.Errorf("Incorrect alarm thresholds")
327 time.Sleep(time.Duration(1) * time.Second)
328 alarmManager.maxActiveAlarms = 5000
329 alarmManager.maxAlarmHistory = 20000
335 func TestGetPrometheusAlerts(t *testing.T) {
336 time.Sleep(1 * time.Second)
337 xapp.Logger.Info("TestGetPrometheusAlerts")
338 ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
340 commandReady := make(chan bool, 1)
341 command := "cli/alarm-cli"
342 args := []string{"gapam", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
343 ExecCLICommand(commandReady, command, args...)
349 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
350 receivedAlert := waitForEvent()
352 assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
353 _, ok := alarmManager.IsMatchFound(a)
359 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
360 receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
361 //assert.Equal(t, receivedAlert, e)
364 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
365 l, err := net.Listen("tcp", "localhost:9093")
367 t.Error("Failed to create listener: " + err.Error())
369 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
370 assert.Equal(t, r.Method, method)
371 assert.Equal(t, r.URL.String(), url)
375 w.Header().Add("Content-Type", "application/json")
376 w.WriteHeader(status)
377 b, _ := json.Marshal(respData)
388 func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
389 l, err := net.Listen("tcp", "localhost:9093")
391 t.Error("Failed to create listener: " + err.Error())
393 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
394 assert.Equal(t, r.Method, method)
395 assert.Equal(t, r.URL.String(), url)
397 w.Header().Add("Content-Type", "application/json")
399 // Read alerts from file
400 payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
402 t.Error("Failed to send response: ", err)
404 _, err = w.Write(payload)
406 t.Error("Failed to send response: " + err.Error())
415 func waitForEvent() string {
416 receivedAlert := <-eventChan
420 func fireEvent(t *testing.T, body io.ReadCloser) {
421 reqBody, err := ioutil.ReadAll(body)
422 assert.Nil(t, err, "ioutil.ReadAll failed")
423 assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
425 eventChan <- fmt.Sprintf("%s", reqBody)
428 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
429 rr := httptest.NewRecorder()
431 handleR.ServeHTTP(rr, req)
436 func checkResponseCode(t *testing.T, expected, actual int) bool {
437 if expected != actual {
438 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
444 func ExecCLICommand(commandReady chan bool, command string, args ...string) {
446 xapp.Logger.Info("Giving CLI command")
447 cmd := exec.Command(command, args...)
449 output, err := cmd.CombinedOutput()
451 xapp.Logger.Info("CLI command failed out: %s", err)
453 xapp.Logger.Info("CLI command output: %s", output)
455 xapp.Logger.Info("CLI command completed")
459 func readJSONFromFile(filename string) ([]byte, error) {
460 file, err := ioutil.ReadFile(filename)
462 err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)