2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
27 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
28 "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
29 "github.com/gorilla/mux"
30 "github.com/prometheus/alertmanager/api/v2/models"
31 "github.com/stretchr/testify/assert"
44 var alarmManager *AlarmManager
45 var alarmer *alarm.RICAlarm
46 var eventChan chan string
49 func TestMain(M *testing.M) {
50 os.Setenv("ALARM_IF_RMR", "true")
51 alarmManager = NewAlarmManager("localhost:9093", 500)
52 go alarmManager.Run(false)
53 time.Sleep(time.Duration(2) * time.Second)
55 // Wait until RMR is up-and-running
56 for !xapp.Rmr.IsReady() {
57 time.Sleep(time.Duration(1) * time.Second)
60 alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
61 alarmManager.alarmClient = alarmer
62 time.Sleep(time.Duration(5) * time.Second)
63 eventChan = make(chan string)
68 func TestSetAlarmDefinitions(t *testing.T) {
69 xapp.Logger.Info("TestSetAlarmDefinitions")
70 var alarm8004Definition alarm.AlarmDefinition
71 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
72 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
73 alarm8004Definition.EventType = "Processing error"
74 alarm8004Definition.OperationInstructions = "Not defined"
76 var alarm8005Definition alarm.AlarmDefinition
77 alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
78 alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
79 alarm8005Definition.EventType = "Communication error"
80 alarm8005Definition.OperationInstructions = "Not defined"
82 var alarm8006Definition alarm.AlarmDefinition
83 alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
84 alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
85 alarm8006Definition.EventType = "Communication error"
86 alarm8006Definition.OperationInstructions = "Not defined"
88 var alarm8007Definition alarm.AlarmDefinition
89 alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
90 alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
91 alarm8007Definition.EventType = "Communication error"
92 alarm8007Definition.OperationInstructions = "Not defined"
94 var alarm8008Definition alarm.AlarmDefinition
95 alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
96 alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
97 alarm8008Definition.EventType = "storage warning"
98 alarm8008Definition.OperationInstructions = "clear alarms or raise threshold"
100 var alarm8009Definition alarm.AlarmDefinition
101 alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
102 alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
103 alarm8009Definition.EventType = "storage warning"
104 alarm8009Definition.OperationInstructions = "clear alarms or raise threshold"
106 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
107 pbodyEn, _ := json.Marshal(pbodyParams)
108 req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
109 handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
110 response := executeRequest(req, handleFunc)
111 status := checkResponseCode(t, http.StatusOK, response.Code)
112 xapp.Logger.Info("status = %v", status)
116 func TestGetAlarmDefinitions(t *testing.T) {
117 xapp.Logger.Info("TestGetAlarmDefinitions")
118 var alarmDefinition alarm.AlarmDefinition
119 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
120 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
121 req = mux.SetURLVars(req, vars)
122 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
123 response := executeRequest(req, handleFunc)
124 checkResponseCode(t, http.StatusOK, response.Code)
125 json.NewDecoder(response.Body).Decode(&alarmDefinition)
126 xapp.Logger.Info("alarm definition = %v", alarmDefinition)
127 if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
128 t.Errorf("Incorrect alarm definition")
132 func TestDeleteAlarmDefinitions(t *testing.T) {
133 xapp.Logger.Info("TestDeleteAlarmDefinitions")
135 var ricAlarmDefinitions RicAlarmDefinitions
136 req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
137 req = mux.SetURLVars(req, nil)
138 handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
139 response := executeRequest(req, handleFunc)
140 checkResponseCode(t, http.StatusOK, response.Code)
141 json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
142 for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
143 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
147 req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
148 vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
149 req = mux.SetURLVars(req, vars)
150 handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
151 response = executeRequest(req, handleFunc)
152 checkResponseCode(t, http.StatusOK, response.Code)
155 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
156 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
157 req = mux.SetURLVars(req, vars)
158 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
159 response = executeRequest(req, handleFunc)
160 checkResponseCode(t, http.StatusBadRequest, response.Code)
163 var alarm8004Definition alarm.AlarmDefinition
164 alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
165 alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
166 alarm8004Definition.EventType = "Processing error"
167 alarm8004Definition.OperationInstructions = "Not defined"
168 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
169 pbodyEn, _ := json.Marshal(pbodyParams)
170 req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
171 handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
172 response = executeRequest(req, handleFunc)
173 checkResponseCode(t, http.StatusOK, response.Code)
176 req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
177 vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
178 req = mux.SetURLVars(req, vars)
179 handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
180 response = executeRequest(req, handleFunc)
181 checkResponseCode(t, http.StatusOK, response.Code)
184 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
185 xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
186 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
189 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
190 assert.Nil(t, alarmer.Raise(a), "raise failed")
195 func TestAlarmClearedSucess(t *testing.T) {
196 xapp.Logger.Info("TestAlarmClearedSucess")
197 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
201 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
202 assert.Nil(t, alarmer.Raise(a), "raise failed")
206 // Now Clear the alarm and check alarm is removed
207 a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
208 assert.Nil(t, alarmer.Clear(a), "clear failed")
210 time.Sleep(time.Duration(2) * time.Second)
211 assert.Equal(t, len(alarmManager.activeAlarms), 0)
214 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
215 xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
216 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
220 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
221 assert.Nil(t, alarmer.Raise(a), "raise failed")
223 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
224 assert.Nil(t, alarmer.Raise(b), "raise failed")
230 func TestMultipleAlarmsClearedSucess(t *testing.T) {
231 xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
232 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
236 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
237 assert.Nil(t, alarmer.Clear(a), "clear failed")
239 b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
240 assert.Nil(t, alarmer.Clear(b), "clear failed")
242 time.Sleep(time.Duration(2) * time.Second)
243 assert.Equal(t, len(alarmManager.activeAlarms), 0)
246 func TestAlarmsSuppresedSucess(t *testing.T) {
247 xapp.Logger.Info("TestAlarmsSuppresedSucess")
248 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
251 // Raise two similar/matching alarms ... the second one suppresed
252 a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
253 assert.Nil(t, alarmer.Raise(a), "raise failed")
254 assert.Nil(t, alarmer.Raise(a), "raise failed")
257 assert.Nil(t, alarmer.Clear(a), "clear failed")
260 func TestInvalidAlarms(t *testing.T) {
261 xapp.Logger.Info("TestInvalidAlarms")
262 a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
263 assert.Nil(t, alarmer.Raise(a), "raise failed")
264 time.Sleep(time.Duration(2) * time.Second)
267 func TestAlarmHandlingErrorCases(t *testing.T) {
268 xapp.Logger.Info("TestAlarmHandlingErrorCases")
269 ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
270 assert.Equal(t, err.Error(), "unexpected end of JSON input")
271 assert.Nil(t, ok, "raise failed")
274 func TestConsumeUnknownMessage(t *testing.T) {
275 xapp.Logger.Info("TestConsumeUnknownMessage")
276 err := alarmManager.Consume(&xapp.RMRParams{})
277 assert.Nil(t, err, "raise failed")
280 func TestStatusCallback(t *testing.T) {
281 xapp.Logger.Info("TestStatusCallback")
282 assert.Equal(t, true, alarmManager.StatusCB())
285 func TestActiveAlarmMaxThresholds(t *testing.T) {
286 xapp.Logger.Info("TestActiveAlarmMaxThresholds")
287 ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
288 alarmManager.maxActiveAlarms = 0
289 alarmManager.maxAlarmHistory = 10
291 a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
292 assert.Nil(t, alarmer.Raise(a), "raise failed")
294 var alarmConfigParams alarm.AlarmConfigParams
295 req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
296 req = mux.SetURLVars(req, nil)
297 handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
298 response := executeRequest(req, handleFunc)
300 // Check HTTP Status Code
301 checkResponseCode(t, http.StatusOK, response.Code)
303 // Decode the json output from handler
304 json.NewDecoder(response.Body).Decode(&alarmConfigParams)
305 if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
306 t.Errorf("Incorrect alarm thresholds")
309 time.Sleep(time.Duration(1) * time.Second)
310 alarmManager.maxActiveAlarms = 5000
311 alarmManager.maxAlarmHistory = 20000
317 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
318 receivedAlert := waitForEvent()
320 assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
321 _, ok := alarmManager.IsMatchFound(a)
327 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
328 receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
329 //assert.Equal(t, receivedAlert, e)
332 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
333 l, err := net.Listen("tcp", "localhost:9093")
335 t.Error("Failed to create listener: " + err.Error())
337 ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
338 assert.Equal(t, r.Method, method)
339 assert.Equal(t, r.URL.String(), url)
343 w.Header().Add("Content-Type", "application/json")
344 w.WriteHeader(status)
345 b, _ := json.Marshal(respData)
356 func waitForEvent() string {
357 receivedAlert := <-eventChan
361 func fireEvent(t *testing.T, body io.ReadCloser) {
362 reqBody, err := ioutil.ReadAll(body)
363 assert.Nil(t, err, "ioutil.ReadAll failed")
364 assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
366 eventChan <- fmt.Sprintf("%s", reqBody)
369 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
370 rr := httptest.NewRecorder()
372 handleR.ServeHTTP(rr, req)
377 func checkResponseCode(t *testing.T, expected, actual int) bool {
378 if expected != actual {
379 t.Errorf("Expected response code %d. Got %d\n", expected, actual)