2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
27 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
28 "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
29 "github.com/gorilla/mux"
30 "github.com/prometheus/alertmanager/api/v2/models"
31 "github.com/stretchr/testify/assert"
44 var alarmManager *AlarmManager
45 var alarmer *alarm.RICAlarm
46 var eventChan chan string
49 func TestMain(M *testing.M) {
50 alarmManager = NewAlarmManager("localhost:9093", 500)
51 alarmManager.alertInterval = 20000
52 go alarmManager.Run(false)
53 time.Sleep(time.Duration(10) * time.Second)
55 // Wait until RMR is up-and-running
56 for !xapp.Rmr.IsReady() {
57 time.Sleep(time.Duration(1) * time.Second)
60 alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
61 alarmManager.alarmClient = alarmer
62 time.Sleep(time.Duration(5) * time.Second)
63 eventChan = make(chan string)
68 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
69 xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
70 var alarmDefinition alarm.AlarmDefinition
71 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
72 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
73 req = mux.SetURLVars(req, vars)
74 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
75 response := executeRequest(req, handleFunc)
76 checkResponseCode(t, http.StatusOK, response.Code)
77 json.NewDecoder(response.Body).Decode(&alarmDefinition)
78 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
79 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
80 t.Errorf("Incorrect alarm definition")
84 func TestSetAlarmDefinitions(t *testing.T) {
85 xapp.Logger.Info("TestSetAlarmDefinitions")
86 var alarm8004Definition alarm.AlarmDefinition
87 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
88 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
89 alarm8004Definition.EventType = "Processing error"
90 alarm8004Definition.OperationInstructions = "Not defined"
92 var alarm8005Definition alarm.AlarmDefinition
93 alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
94 alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
95 alarm8005Definition.EventType = "Communication error"
96 alarm8005Definition.OperationInstructions = "Not defined"
98 var alarm8006Definition alarm.AlarmDefinition
99 alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
100 alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
101 alarm8006Definition.EventType = "Communication error"
102 alarm8006Definition.OperationInstructions = "Not defined"
104 var alarm8007Definition alarm.AlarmDefinition
105 alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
106 alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
107 alarm8007Definition.EventType = "Communication error"
108 alarm8007Definition.OperationInstructions = "Not defined"
110 var alarm8008Definition alarm.AlarmDefinition
111 alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
112 alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
113 alarm8008Definition.EventType = "storage warning"
114 alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
116 var alarm8009Definition alarm.AlarmDefinition
117 alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
118 alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
119 alarm8009Definition.EventType = "storage warning"
120 alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
122 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
123 pbodyEn, _ := json.Marshal(pbodyParams)
124 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
125 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
126 response := executeRequest(req, handleFunc)
127 status := checkResponseCode(t, http.StatusOK, response.Code)
128 xapp.Logger.Info("status = %v", status)
132 func TestGetAlarmDefinitions(t *testing.T) {
133 xapp.Logger.Info("TestGetAlarmDefinitions")
134 var alarmDefinition alarm.AlarmDefinition
135 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
136 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
137 req = mux.SetURLVars(req, vars)
138 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
139 response := executeRequest(req, handleFunc)
140 checkResponseCode(t, http.StatusOK, response.Code)
141 json.NewDecoder(response.Body).Decode(&alarmDefinition)
142 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
143 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
144 t.Errorf("Incorrect alarm definition")
148 func TestDeleteAlarmDefinitions(t *testing.T) {
149 xapp.Logger.Info("TestDeleteAlarmDefinitions")
151 var ricAlarmDefinitions RicAlarmDefinitions
152 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
153 req = mux.SetURLVars(req, nil)
154 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
155 response := executeRequest(req, handleFunc)
156 checkResponseCode(t, http.StatusOK, response.Code)
157 json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
158 for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
159 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
163 req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
164 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
165 req = mux.SetURLVars(req, vars)
166 handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
167 response = executeRequest(req, handleFunc)
168 checkResponseCode(t, http.StatusOK, response.Code)
171 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
172 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
173 req = mux.SetURLVars(req, vars)
174 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
175 response = executeRequest(req, handleFunc)
176 checkResponseCode(t, http.StatusBadRequest, response.Code)
179 var alarm8004Definition alarm.AlarmDefinition
180 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
181 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
182 alarm8004Definition.EventType = "Processing error"
183 alarm8004Definition.OperationInstructions = "Not defined"
184 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
185 pbodyEn, _ := json.Marshal(pbodyParams)
186 req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
187 handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
188 response = executeRequest(req, handleFunc)
189 checkResponseCode(t, http.StatusOK, response.Code)
192 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
193 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
194 req = mux.SetURLVars(req, vars)
195 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
196 response = executeRequest(req, handleFunc)
197 checkResponseCode(t, http.StatusOK, response.Code)
200 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
201 xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
202 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
205 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
206 assert.Nil(t, alarmer.Raise(a), "raise failed")
211 func TestAlarmClearedSucess(t *testing.T) {
212 xapp.Logger.Info("TestAlarmClearedSucess")
213 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
217 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
218 assert.Nil(t, alarmer.Raise(a), "raise failed")
222 // Now Clear the alarm and check alarm is removed
223 a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
224 assert.Nil(t, alarmer.Clear(a), "clear failed")
226 time.Sleep(time.Duration(2) * time.Second)
227 assert.Equal(t, len(alarmManager.activeAlarms), 0)
230 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
231 xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
232 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
236 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
237 assert.Nil(t, alarmer.Raise(a), "raise failed")
239 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
240 assert.Nil(t, alarmer.Raise(b), "raise failed")
242 time.Sleep(time.Duration(2) * time.Second)
247 func TestMultipleAlarmsClearedSucess(t *testing.T) {
248 xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
249 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
253 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
254 assert.Nil(t, alarmer.Clear(a), "clear failed")
256 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
257 assert.Nil(t, alarmer.Clear(b), "clear failed")
259 time.Sleep(time.Duration(2) * time.Second)
260 assert.Equal(t, len(alarmManager.activeAlarms), 0)
263 func TestAlarmsSuppresedSucess(t *testing.T) {
264 xapp.Logger.Info("TestAlarmsSuppresedSucess")
265 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
268 // Raise two similar/matching alarms ... the second one suppresed
269 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
270 assert.Nil(t, alarmer.Raise(a), "raise failed")
271 assert.Nil(t, alarmer.Raise(a), "raise failed")
274 assert.Nil(t, alarmer.Clear(a), "clear failed")
277 func TestInvalidAlarms(t *testing.T) {
278 xapp.Logger.Info("TestInvalidAlarms")
279 a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
280 assert.Nil(t, alarmer.Raise(a), "raise failed")
281 time.Sleep(time.Duration(2) * time.Second)
284 func TestAlarmHandlingErrorCases(t *testing.T) {
285 xapp.Logger.Info("TestAlarmHandlingErrorCases")
286 ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
287 assert.Equal(t, err.Error(), "unexpected end of JSON input")
288 assert.Nil(t, ok, "raise failed")
291 func TestConsumeUnknownMessage(t *testing.T) {
292 xapp.Logger.Info("TestConsumeUnknownMessage")
293 err := alarmManager.Consume(&xapp.RMRParams{})
294 assert.Nil(t, err, "raise failed")
297 func TestStatusCallback(t *testing.T) {
298 xapp.Logger.Info("TestStatusCallback")
299 assert.Equal(t, true, alarmManager.StatusCB())
302 func TestActiveAlarmMaxThresholds(t *testing.T) {
303 xapp.Logger.Info("TestActiveAlarmMaxThresholds")
304 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
305 alarmManager.maxActiveAlarms = 0
306 alarmManager.maxAlarmHistory = 10
308 a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
309 assert.Nil(t, alarmer.Raise(a), "raise failed")
311 var alarmConfigParams alarm.AlarmConfigParams
312 req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
313 req = mux.SetURLVars(req, nil)
314 handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
315 response := executeRequest(req, handleFunc)
317 // Check HTTP Status Code
318 checkResponseCode(t, http.StatusOK, response.Code)
320 // Decode the json output from handler
321 json.NewDecoder(response.Body).Decode(&alarmConfigParams)
322 if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
323 t.Errorf("Incorrect alarm thresholds")
326 time.Sleep(time.Duration(1) * time.Second)
327 alarmManager.maxActiveAlarms = 5000
328 alarmManager.maxAlarmHistory = 20000
334 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
335 receivedAlert := waitForEvent()
337 assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
338 _, ok := alarmManager.IsMatchFound(a)
344 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
345 receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
346 //assert.Equal(t, receivedAlert, e)
349 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
350 l, err := net.Listen("tcp", "localhost:9093")
352 t.Error("Failed to create listener: " + err.Error())
354 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
355 assert.Equal(t, r.Method, method)
356 assert.Equal(t, r.URL.String(), url)
360 w.Header().Add("Content-Type", "application/json")
361 w.WriteHeader(status)
362 b, _ := json.Marshal(respData)
373 func waitForEvent() string {
374 receivedAlert := <-eventChan
378 func fireEvent(t *testing.T, body io.ReadCloser) {
379 reqBody, err := ioutil.ReadAll(body)
380 assert.Nil(t, err, "ioutil.ReadAll failed")
381 assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
383 eventChan <- fmt.Sprintf("%s", reqBody)
386 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
387 rr := httptest.NewRecorder()
389 handleR.ServeHTTP(rr, req)
394 func checkResponseCode(t *testing.T, expected, actual int) bool {
395 if expected != actual {
396 t.Errorf("Expected response code %d. Got %d\n", expected, actual)