2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
27 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
28 "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
29 "github.com/gorilla/mux"
30 "github.com/prometheus/alertmanager/api/v2/models"
31 "github.com/stretchr/testify/assert"
44 var alarmManager *AlarmManager
45 var alarmer *alarm.RICAlarm
46 var eventChan chan string
49 func TestMain(M *testing.M) {
50 alarmManager = NewAlarmManager("localhost:9093", 500)
51 go alarmManager.Run(false)
52 time.Sleep(time.Duration(10) * time.Second)
54 // Wait until RMR is up-and-running
55 for !xapp.Rmr.IsReady() {
56 time.Sleep(time.Duration(1) * time.Second)
59 alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
60 alarmManager.alarmClient = alarmer
61 time.Sleep(time.Duration(5) * time.Second)
62 eventChan = make(chan string)
67 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
68 xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
69 var alarmDefinition alarm.AlarmDefinition
70 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
71 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
72 req = mux.SetURLVars(req, vars)
73 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
74 response := executeRequest(req, handleFunc)
75 checkResponseCode(t, http.StatusOK, response.Code)
76 json.NewDecoder(response.Body).Decode(&alarmDefinition)
77 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
78 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
79 t.Errorf("Incorrect alarm definition")
83 func TestSetAlarmDefinitions(t *testing.T) {
84 xapp.Logger.Info("TestSetAlarmDefinitions")
85 var alarm8004Definition alarm.AlarmDefinition
86 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
87 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
88 alarm8004Definition.EventType = "Processing error"
89 alarm8004Definition.OperationInstructions = "Not defined"
91 var alarm8005Definition alarm.AlarmDefinition
92 alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
93 alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
94 alarm8005Definition.EventType = "Communication error"
95 alarm8005Definition.OperationInstructions = "Not defined"
97 var alarm8006Definition alarm.AlarmDefinition
98 alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
99 alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
100 alarm8006Definition.EventType = "Communication error"
101 alarm8006Definition.OperationInstructions = "Not defined"
103 var alarm8007Definition alarm.AlarmDefinition
104 alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
105 alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
106 alarm8007Definition.EventType = "Communication error"
107 alarm8007Definition.OperationInstructions = "Not defined"
109 var alarm8008Definition alarm.AlarmDefinition
110 alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
111 alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
112 alarm8008Definition.EventType = "storage warning"
113 alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
115 var alarm8009Definition alarm.AlarmDefinition
116 alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
117 alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
118 alarm8009Definition.EventType = "storage warning"
119 alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
121 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
122 pbodyEn, _ := json.Marshal(pbodyParams)
123 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
124 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
125 response := executeRequest(req, handleFunc)
126 status := checkResponseCode(t, http.StatusOK, response.Code)
127 xapp.Logger.Info("status = %v", status)
131 func TestGetAlarmDefinitions(t *testing.T) {
132 xapp.Logger.Info("TestGetAlarmDefinitions")
133 var alarmDefinition alarm.AlarmDefinition
134 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
135 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
136 req = mux.SetURLVars(req, vars)
137 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
138 response := executeRequest(req, handleFunc)
139 checkResponseCode(t, http.StatusOK, response.Code)
140 json.NewDecoder(response.Body).Decode(&alarmDefinition)
141 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
142 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
143 t.Errorf("Incorrect alarm definition")
147 func TestDeleteAlarmDefinitions(t *testing.T) {
148 xapp.Logger.Info("TestDeleteAlarmDefinitions")
150 var ricAlarmDefinitions RicAlarmDefinitions
151 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
152 req = mux.SetURLVars(req, nil)
153 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
154 response := executeRequest(req, handleFunc)
155 checkResponseCode(t, http.StatusOK, response.Code)
156 json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
157 for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
158 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
162 req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
163 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
164 req = mux.SetURLVars(req, vars)
165 handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
166 response = executeRequest(req, handleFunc)
167 checkResponseCode(t, http.StatusOK, response.Code)
170 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
171 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
172 req = mux.SetURLVars(req, vars)
173 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
174 response = executeRequest(req, handleFunc)
175 checkResponseCode(t, http.StatusBadRequest, response.Code)
178 var alarm8004Definition alarm.AlarmDefinition
179 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
180 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
181 alarm8004Definition.EventType = "Processing error"
182 alarm8004Definition.OperationInstructions = "Not defined"
183 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
184 pbodyEn, _ := json.Marshal(pbodyParams)
185 req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
186 handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
187 response = executeRequest(req, handleFunc)
188 checkResponseCode(t, http.StatusOK, response.Code)
191 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
192 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
193 req = mux.SetURLVars(req, vars)
194 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
195 response = executeRequest(req, handleFunc)
196 checkResponseCode(t, http.StatusOK, response.Code)
199 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
200 xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
201 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
204 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
205 assert.Nil(t, alarmer.Raise(a), "raise failed")
210 func TestAlarmClearedSucess(t *testing.T) {
211 xapp.Logger.Info("TestAlarmClearedSucess")
212 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
216 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
217 assert.Nil(t, alarmer.Raise(a), "raise failed")
221 // Now Clear the alarm and check alarm is removed
222 a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
223 assert.Nil(t, alarmer.Clear(a), "clear failed")
225 time.Sleep(time.Duration(2) * time.Second)
226 assert.Equal(t, len(alarmManager.activeAlarms), 0)
229 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
230 xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
231 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
235 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
236 assert.Nil(t, alarmer.Raise(a), "raise failed")
238 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
239 assert.Nil(t, alarmer.Raise(b), "raise failed")
245 func TestMultipleAlarmsClearedSucess(t *testing.T) {
246 xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
247 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
251 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
252 assert.Nil(t, alarmer.Clear(a), "clear failed")
254 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
255 assert.Nil(t, alarmer.Clear(b), "clear failed")
257 time.Sleep(time.Duration(2) * time.Second)
258 assert.Equal(t, len(alarmManager.activeAlarms), 0)
261 func TestAlarmsSuppresedSucess(t *testing.T) {
262 xapp.Logger.Info("TestAlarmsSuppresedSucess")
263 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
266 // Raise two similar/matching alarms ... the second one suppresed
267 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
268 assert.Nil(t, alarmer.Raise(a), "raise failed")
269 assert.Nil(t, alarmer.Raise(a), "raise failed")
272 assert.Nil(t, alarmer.Clear(a), "clear failed")
275 func TestInvalidAlarms(t *testing.T) {
276 xapp.Logger.Info("TestInvalidAlarms")
277 a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
278 assert.Nil(t, alarmer.Raise(a), "raise failed")
279 time.Sleep(time.Duration(2) * time.Second)
282 func TestAlarmHandlingErrorCases(t *testing.T) {
283 xapp.Logger.Info("TestAlarmHandlingErrorCases")
284 ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
285 assert.Equal(t, err.Error(), "unexpected end of JSON input")
286 assert.Nil(t, ok, "raise failed")
289 func TestConsumeUnknownMessage(t *testing.T) {
290 xapp.Logger.Info("TestConsumeUnknownMessage")
291 err := alarmManager.Consume(&xapp.RMRParams{})
292 assert.Nil(t, err, "raise failed")
295 func TestStatusCallback(t *testing.T) {
296 xapp.Logger.Info("TestStatusCallback")
297 assert.Equal(t, true, alarmManager.StatusCB())
300 func TestActiveAlarmMaxThresholds(t *testing.T) {
301 xapp.Logger.Info("TestActiveAlarmMaxThresholds")
302 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
303 alarmManager.maxActiveAlarms = 0
304 alarmManager.maxAlarmHistory = 10
306 a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
307 assert.Nil(t, alarmer.Raise(a), "raise failed")
309 var alarmConfigParams alarm.AlarmConfigParams
310 req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
311 req = mux.SetURLVars(req, nil)
312 handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
313 response := executeRequest(req, handleFunc)
315 // Check HTTP Status Code
316 checkResponseCode(t, http.StatusOK, response.Code)
318 // Decode the json output from handler
319 json.NewDecoder(response.Body).Decode(&alarmConfigParams)
320 if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
321 t.Errorf("Incorrect alarm thresholds")
324 time.Sleep(time.Duration(1) * time.Second)
325 alarmManager.maxActiveAlarms = 5000
326 alarmManager.maxAlarmHistory = 20000
332 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
333 receivedAlert := waitForEvent()
335 assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
336 _, ok := alarmManager.IsMatchFound(a)
342 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
343 receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
344 //assert.Equal(t, receivedAlert, e)
347 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
348 l, err := net.Listen("tcp", "localhost:9093")
350 t.Error("Failed to create listener: " + err.Error())
352 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
353 assert.Equal(t, r.Method, method)
354 assert.Equal(t, r.URL.String(), url)
358 w.Header().Add("Content-Type", "application/json")
359 w.WriteHeader(status)
360 b, _ := json.Marshal(respData)
371 func waitForEvent() string {
372 receivedAlert := <-eventChan
376 func fireEvent(t *testing.T, body io.ReadCloser) {
377 reqBody, err := ioutil.ReadAll(body)
378 assert.Nil(t, err, "ioutil.ReadAll failed")
379 assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
381 eventChan <- fmt.Sprintf("%s", reqBody)
384 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
385 rr := httptest.NewRecorder()
387 handleR.ServeHTTP(rr, req)
392 func checkResponseCode(t *testing.T, expected, actual int) bool {
393 if expected != actual {
394 t.Errorf("Expected response code %d. Got %d\n", expected, actual)