2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
27 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
28 "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
29 "github.com/gorilla/mux"
30 "github.com/prometheus/alertmanager/api/v2/models"
31 "github.com/stretchr/testify/assert"
44 var alarmManager *AlarmManager
45 var alarmer *alarm.RICAlarm
46 var eventChan chan string
49 func TestMain(M *testing.M) {
50 os.Setenv("ALARM_IF_RMR", "true")
51 alarmManager = NewAlarmManager("localhost:9093", 500)
52 go alarmManager.Run(false)
53 time.Sleep(time.Duration(10) * time.Second)
55 // Wait until RMR is up-and-running
56 for !xapp.Rmr.IsReady() {
57 time.Sleep(time.Duration(1) * time.Second)
60 alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
61 alarmManager.alarmClient = alarmer
62 time.Sleep(time.Duration(5) * time.Second)
63 eventChan = make(chan string)
68 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
69 xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
70 var alarmDefinition alarm.AlarmDefinition
71 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
72 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
73 req = mux.SetURLVars(req, vars)
74 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
75 response := executeRequest(req, handleFunc)
76 checkResponseCode(t, http.StatusOK, response.Code)
77 json.NewDecoder(response.Body).Decode(&alarmDefinition)
78 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
79 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
80 t.Errorf("Incorrect alarm definition")
84 func TestSetAlarmDefinitions(t *testing.T) {
85 xapp.Logger.Info("TestSetAlarmDefinitions")
86 var alarm8004Definition alarm.AlarmDefinition
87 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
88 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
89 alarm8004Definition.EventType = "Processing error"
90 alarm8004Definition.OperationInstructions = "Not defined"
92 var alarm8005Definition alarm.AlarmDefinition
93 alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
94 alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
95 alarm8005Definition.EventType = "Communication error"
96 alarm8005Definition.OperationInstructions = "Not defined"
98 var alarm8006Definition alarm.AlarmDefinition
99 alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
100 alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
101 alarm8006Definition.EventType = "Communication error"
102 alarm8006Definition.OperationInstructions = "Not defined"
104 var alarm8007Definition alarm.AlarmDefinition
105 alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
106 alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
107 alarm8007Definition.EventType = "Communication error"
108 alarm8007Definition.OperationInstructions = "Not defined"
110 var alarm8008Definition alarm.AlarmDefinition
111 alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
112 alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
113 alarm8008Definition.EventType = "storage warning"
114 alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
116 var alarm8009Definition alarm.AlarmDefinition
117 alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
118 alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
119 alarm8009Definition.EventType = "storage warning"
120 alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
122 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
123 pbodyEn, _ := json.Marshal(pbodyParams)
124 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
125 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
126 response := executeRequest(req, handleFunc)
127 status := checkResponseCode(t, http.StatusOK, response.Code)
128 xapp.Logger.Info("status = %v", status)
132 func TestGetAlarmDefinitions(t *testing.T) {
133 xapp.Logger.Info("TestGetAlarmDefinitions")
134 var alarmDefinition alarm.AlarmDefinition
135 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
136 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
137 req = mux.SetURLVars(req, vars)
138 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
139 response := executeRequest(req, handleFunc)
140 checkResponseCode(t, http.StatusOK, response.Code)
141 json.NewDecoder(response.Body).Decode(&alarmDefinition)
142 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
143 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
144 t.Errorf("Incorrect alarm definition")
148 func TestDeleteAlarmDefinitions(t *testing.T) {
149 xapp.Logger.Info("TestDeleteAlarmDefinitions")
151 var ricAlarmDefinitions RicAlarmDefinitions
152 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
153 req = mux.SetURLVars(req, nil)
154 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
155 response := executeRequest(req, handleFunc)
156 checkResponseCode(t, http.StatusOK, response.Code)
157 json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
158 for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
159 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
163 req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
164 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
165 req = mux.SetURLVars(req, vars)
166 handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
167 response = executeRequest(req, handleFunc)
168 checkResponseCode(t, http.StatusOK, response.Code)
171 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
172 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
173 req = mux.SetURLVars(req, vars)
174 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
175 response = executeRequest(req, handleFunc)
176 checkResponseCode(t, http.StatusBadRequest, response.Code)
179 var alarm8004Definition alarm.AlarmDefinition
180 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
181 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
182 alarm8004Definition.EventType = "Processing error"
183 alarm8004Definition.OperationInstructions = "Not defined"
184 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
185 pbodyEn, _ := json.Marshal(pbodyParams)
186 req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
187 handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
188 response = executeRequest(req, handleFunc)
189 checkResponseCode(t, http.StatusOK, response.Code)
192 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
193 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
194 req = mux.SetURLVars(req, vars)
195 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
196 response = executeRequest(req, handleFunc)
197 checkResponseCode(t, http.StatusOK, response.Code)
200 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
201 xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
202 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
205 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
206 assert.Nil(t, alarmer.Raise(a), "raise failed")
211 func TestAlarmClearedSucess(t *testing.T) {
212 xapp.Logger.Info("TestAlarmClearedSucess")
213 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
217 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
218 assert.Nil(t, alarmer.Raise(a), "raise failed")
222 // Now Clear the alarm and check alarm is removed
223 a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
224 assert.Nil(t, alarmer.Clear(a), "clear failed")
226 time.Sleep(time.Duration(2) * time.Second)
227 assert.Equal(t, len(alarmManager.activeAlarms), 0)
230 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
231 xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
232 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
236 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
237 assert.Nil(t, alarmer.Raise(a), "raise failed")
239 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
240 assert.Nil(t, alarmer.Raise(b), "raise failed")
246 func TestMultipleAlarmsClearedSucess(t *testing.T) {
247 xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
248 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
252 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
253 assert.Nil(t, alarmer.Clear(a), "clear failed")
255 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
256 assert.Nil(t, alarmer.Clear(b), "clear failed")
258 time.Sleep(time.Duration(2) * time.Second)
259 assert.Equal(t, len(alarmManager.activeAlarms), 0)
262 func TestAlarmsSuppresedSucess(t *testing.T) {
263 xapp.Logger.Info("TestAlarmsSuppresedSucess")
264 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
267 // Raise two similar/matching alarms ... the second one suppresed
268 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
269 assert.Nil(t, alarmer.Raise(a), "raise failed")
270 assert.Nil(t, alarmer.Raise(a), "raise failed")
273 assert.Nil(t, alarmer.Clear(a), "clear failed")
276 func TestInvalidAlarms(t *testing.T) {
277 xapp.Logger.Info("TestInvalidAlarms")
278 a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
279 assert.Nil(t, alarmer.Raise(a), "raise failed")
280 time.Sleep(time.Duration(2) * time.Second)
283 func TestAlarmHandlingErrorCases(t *testing.T) {
284 xapp.Logger.Info("TestAlarmHandlingErrorCases")
285 ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
286 assert.Equal(t, err.Error(), "unexpected end of JSON input")
287 assert.Nil(t, ok, "raise failed")
290 func TestConsumeUnknownMessage(t *testing.T) {
291 xapp.Logger.Info("TestConsumeUnknownMessage")
292 err := alarmManager.Consume(&xapp.RMRParams{})
293 assert.Nil(t, err, "raise failed")
296 func TestStatusCallback(t *testing.T) {
297 xapp.Logger.Info("TestStatusCallback")
298 assert.Equal(t, true, alarmManager.StatusCB())
301 func TestActiveAlarmMaxThresholds(t *testing.T) {
302 xapp.Logger.Info("TestActiveAlarmMaxThresholds")
303 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
304 alarmManager.maxActiveAlarms = 0
305 alarmManager.maxAlarmHistory = 10
307 a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
308 assert.Nil(t, alarmer.Raise(a), "raise failed")
310 var alarmConfigParams alarm.AlarmConfigParams
311 req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
312 req = mux.SetURLVars(req, nil)
313 handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
314 response := executeRequest(req, handleFunc)
316 // Check HTTP Status Code
317 checkResponseCode(t, http.StatusOK, response.Code)
319 // Decode the json output from handler
320 json.NewDecoder(response.Body).Decode(&alarmConfigParams)
321 if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
322 t.Errorf("Incorrect alarm thresholds")
325 time.Sleep(time.Duration(1) * time.Second)
326 alarmManager.maxActiveAlarms = 5000
327 alarmManager.maxAlarmHistory = 20000
333 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
334 receivedAlert := waitForEvent()
336 assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
337 _, ok := alarmManager.IsMatchFound(a)
343 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
344 receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
345 //assert.Equal(t, receivedAlert, e)
348 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
349 l, err := net.Listen("tcp", "localhost:9093")
351 t.Error("Failed to create listener: " + err.Error())
353 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
354 assert.Equal(t, r.Method, method)
355 assert.Equal(t, r.URL.String(), url)
359 w.Header().Add("Content-Type", "application/json")
360 w.WriteHeader(status)
361 b, _ := json.Marshal(respData)
372 func waitForEvent() string {
373 receivedAlert := <-eventChan
377 func fireEvent(t *testing.T, body io.ReadCloser) {
378 reqBody, err := ioutil.ReadAll(body)
379 assert.Nil(t, err, "ioutil.ReadAll failed")
380 assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
382 eventChan <- fmt.Sprintf("%s", reqBody)
385 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
386 rr := httptest.NewRecorder()
388 handleR.ServeHTTP(rr, req)
393 func checkResponseCode(t *testing.T, expected, actual int) bool {
394 if expected != actual {
395 t.Errorf("Expected response code %d. Got %d\n", expected, actual)