f4a5aee9de1272119b4fdcf361daf085919485dc
[ric-plt/alarm-go.git] / manager / cmd / manager_test.go
1 /*
2  *  Copyright (c) 2020 AT&T Intellectual Property.
3  *  Copyright (c) 2020 Nokia.
4  *
5  *  Licensed under the Apache License, Version 2.0 (the "License");
6  *  you may not use this file except in compliance with the License.
7  *  You may obtain a copy of the License at
8  *
9  *     http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *  Unless required by applicable law or agreed to in writing, software
12  *  distributed under the License is distributed on an "AS IS" BASIS,
13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *  See the License for the specific language governing permissions and
15  *  limitations under the License.
16  *
17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18  * platform project (RICP).
19  */
20
21 package main
22
23 import (
24         "bytes"
25         "encoding/json"
26         "fmt"
27         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
28         "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
29         "github.com/gorilla/mux"
30         "github.com/prometheus/alertmanager/api/v2/models"
31         "github.com/stretchr/testify/assert"
32         "io"
33         "io/ioutil"
34         "net"
35         "net/http"
36         "net/http/httptest"
37         "os"
38         "strconv"
39         "strings"
40         "testing"
41         "time"
42 )
43
44 var alarmManager *AlarmManager
45 var alarmer *alarm.RICAlarm
46 var eventChan chan string
47
48 // Test cases
49 func TestMain(M *testing.M) {
50         alarmManager = NewAlarmManager("localhost:9093", 500)
51         go alarmManager.Run(false)
52         time.Sleep(time.Duration(10) * time.Second)
53
54         // Wait until RMR is up-and-running
55         for !xapp.Rmr.IsReady() {
56                 time.Sleep(time.Duration(1) * time.Second)
57         }
58
59         alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
60         alarmManager.alarmClient = alarmer
61         time.Sleep(time.Duration(5) * time.Second)
62         eventChan = make(chan string)
63
64         os.Exit(M.Run())
65 }
66
67 func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
68         xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
69         var alarmDefinition alarm.AlarmDefinition
70         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
71         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
72         req = mux.SetURLVars(req, vars)
73         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
74         response := executeRequest(req, handleFunc)
75         checkResponseCode(t, http.StatusOK, response.Code)
76         json.NewDecoder(response.Body).Decode(&alarmDefinition)
77         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
78         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
79                 t.Errorf("Incorrect alarm definition")
80         }
81 }
82
83 func TestSetAlarmDefinitions(t *testing.T) {
84         xapp.Logger.Info("TestSetAlarmDefinitions")
85         var alarm8004Definition alarm.AlarmDefinition
86         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
87         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
88         alarm8004Definition.EventType = "Processing error"
89         alarm8004Definition.OperationInstructions = "Not defined"
90
91         var alarm8005Definition alarm.AlarmDefinition
92         alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
93         alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
94         alarm8005Definition.EventType = "Communication error"
95         alarm8005Definition.OperationInstructions = "Not defined"
96
97         var alarm8006Definition alarm.AlarmDefinition
98         alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
99         alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
100         alarm8006Definition.EventType = "Communication error"
101         alarm8006Definition.OperationInstructions = "Not defined"
102
103         var alarm8007Definition alarm.AlarmDefinition
104         alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
105         alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
106         alarm8007Definition.EventType = "Communication error"
107         alarm8007Definition.OperationInstructions = "Not defined"
108
109         var alarm8008Definition alarm.AlarmDefinition
110         alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
111         alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
112         alarm8008Definition.EventType = "storage warning"
113         alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
114
115         var alarm8009Definition alarm.AlarmDefinition
116         alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
117         alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
118         alarm8009Definition.EventType = "storage warning"
119         alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
120
121         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
122         pbodyEn, _ := json.Marshal(pbodyParams)
123         req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
124         handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
125         response := executeRequest(req, handleFunc)
126         status := checkResponseCode(t, http.StatusOK, response.Code)
127         xapp.Logger.Info("status = %v", status)
128
129 }
130
131 func TestGetAlarmDefinitions(t *testing.T) {
132         xapp.Logger.Info("TestGetAlarmDefinitions")
133         var alarmDefinition alarm.AlarmDefinition
134         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
135         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
136         req = mux.SetURLVars(req, vars)
137         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
138         response := executeRequest(req, handleFunc)
139         checkResponseCode(t, http.StatusOK, response.Code)
140         json.NewDecoder(response.Body).Decode(&alarmDefinition)
141         xapp.Logger.Info("alarm definition = %v", alarmDefinition)
142         if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
143                 t.Errorf("Incorrect alarm definition")
144         }
145 }
146
147 func TestDeleteAlarmDefinitions(t *testing.T) {
148         xapp.Logger.Info("TestDeleteAlarmDefinitions")
149         //Get all
150         var ricAlarmDefinitions RicAlarmDefinitions
151         req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
152         req = mux.SetURLVars(req, nil)
153         handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
154         response := executeRequest(req, handleFunc)
155         checkResponseCode(t, http.StatusOK, response.Code)
156         json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
157         for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
158                 xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
159         }
160
161         //Delete 8004
162         req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
163         vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
164         req = mux.SetURLVars(req, vars)
165         handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
166         response = executeRequest(req, handleFunc)
167         checkResponseCode(t, http.StatusOK, response.Code)
168
169         //Get 8004 fail
170         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
171         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
172         req = mux.SetURLVars(req, vars)
173         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
174         response = executeRequest(req, handleFunc)
175         checkResponseCode(t, http.StatusBadRequest, response.Code)
176
177         //Set 8004 success
178         var alarm8004Definition alarm.AlarmDefinition
179         alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
180         alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
181         alarm8004Definition.EventType = "Processing error"
182         alarm8004Definition.OperationInstructions = "Not defined"
183         pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
184         pbodyEn, _ := json.Marshal(pbodyParams)
185         req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
186         handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
187         response = executeRequest(req, handleFunc)
188         checkResponseCode(t, http.StatusOK, response.Code)
189
190         //Get 8004 success
191         req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
192         vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
193         req = mux.SetURLVars(req, vars)
194         handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
195         response = executeRequest(req, handleFunc)
196         checkResponseCode(t, http.StatusOK, response.Code)
197 }
198
199 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
200         xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
201         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
202         defer ts.Close()
203
204         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
205         assert.Nil(t, alarmer.Raise(a), "raise failed")
206
207         VerifyAlarm(t, a, 1)
208 }
209
210 func TestAlarmClearedSucess(t *testing.T) {
211         xapp.Logger.Info("TestAlarmClearedSucess")
212         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
213         defer ts.Close()
214
215         // Raise the alarm
216         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
217         assert.Nil(t, alarmer.Raise(a), "raise failed")
218
219         VerifyAlarm(t, a, 1)
220
221         // Now Clear the alarm and check alarm is removed
222         a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1")
223         assert.Nil(t, alarmer.Clear(a), "clear failed")
224
225         time.Sleep(time.Duration(2) * time.Second)
226         assert.Equal(t, len(alarmManager.activeAlarms), 0)
227 }
228
229 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
230         xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
231         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
232         defer ts.Close()
233
234         // Raise two alarms
235         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
236         assert.Nil(t, alarmer.Raise(a), "raise failed")
237
238         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
239         assert.Nil(t, alarmer.Raise(b), "raise failed")
240
241         VerifyAlarm(t, a, 2)
242         VerifyAlarm(t, b, 2)
243 }
244
245 func TestMultipleAlarmsClearedSucess(t *testing.T) {
246         xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
247         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
248         defer ts.Close()
249
250         // Raise two alarms
251         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
252         assert.Nil(t, alarmer.Clear(a), "clear failed")
253
254         b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
255         assert.Nil(t, alarmer.Clear(b), "clear failed")
256
257         time.Sleep(time.Duration(2) * time.Second)
258         assert.Equal(t, len(alarmManager.activeAlarms), 0)
259 }
260
261 func TestAlarmsSuppresedSucess(t *testing.T) {
262         xapp.Logger.Info("TestAlarmsSuppresedSucess")
263         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
264         defer ts.Close()
265
266         // Raise two similar/matching alarms ... the second one suppresed
267         a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
268         assert.Nil(t, alarmer.Raise(a), "raise failed")
269         assert.Nil(t, alarmer.Raise(a), "raise failed")
270
271         VerifyAlarm(t, a, 1)
272         assert.Nil(t, alarmer.Clear(a), "clear failed")
273 }
274
275 func TestInvalidAlarms(t *testing.T) {
276         xapp.Logger.Info("TestInvalidAlarms")
277         a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
278         assert.Nil(t, alarmer.Raise(a), "raise failed")
279         time.Sleep(time.Duration(2) * time.Second)
280 }
281
282 func TestAlarmHandlingErrorCases(t *testing.T) {
283         xapp.Logger.Info("TestAlarmHandlingErrorCases")
284         ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
285         assert.Equal(t, err.Error(), "unexpected end of JSON input")
286         assert.Nil(t, ok, "raise failed")
287 }
288
289 func TestConsumeUnknownMessage(t *testing.T) {
290         xapp.Logger.Info("TestConsumeUnknownMessage")
291         err := alarmManager.Consume(&xapp.RMRParams{})
292         assert.Nil(t, err, "raise failed")
293 }
294
295 func TestStatusCallback(t *testing.T) {
296         xapp.Logger.Info("TestStatusCallback")
297         assert.Equal(t, true, alarmManager.StatusCB())
298 }
299
300 func TestActiveAlarmMaxThresholds(t *testing.T) {
301         xapp.Logger.Info("TestActiveAlarmMaxThresholds")
302         ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
303         alarmManager.maxActiveAlarms = 0
304         alarmManager.maxAlarmHistory = 10
305
306         a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
307         assert.Nil(t, alarmer.Raise(a), "raise failed")
308
309         var alarmConfigParams alarm.AlarmConfigParams
310         req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
311         req = mux.SetURLVars(req, nil)
312         handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
313         response := executeRequest(req, handleFunc)
314
315         // Check HTTP Status Code
316         checkResponseCode(t, http.StatusOK, response.Code)
317
318         // Decode the json output from handler
319         json.NewDecoder(response.Body).Decode(&alarmConfigParams)
320         if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
321                 t.Errorf("Incorrect alarm thresholds")
322         }
323
324         time.Sleep(time.Duration(1) * time.Second)
325         alarmManager.maxActiveAlarms = 5000
326         alarmManager.maxAlarmHistory = 20000
327         VerifyAlarm(t, a, 2)
328         VerifyAlarm(t, a, 2)
329         ts.Close()
330 }
331
332 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
333         receivedAlert := waitForEvent()
334
335         assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
336         _, ok := alarmManager.IsMatchFound(a)
337         assert.True(t, ok)
338
339         return receivedAlert
340 }
341
342 func VerifyAlert(t *testing.T, receivedAlert, expectedAlert string) {
343         receivedAlert = strings.Replace(fmt.Sprintf("%v", receivedAlert), "\r\n", " ", -1)
344         //assert.Equal(t, receivedAlert, e)
345 }
346
347 func CreatePromAlertSimulator(t *testing.T, method, url string, status int, respData interface{}) *httptest.Server {
348         l, err := net.Listen("tcp", "localhost:9093")
349         if err != nil {
350                 t.Error("Failed to create listener: " + err.Error())
351         }
352         ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
353                 assert.Equal(t, r.Method, method)
354                 assert.Equal(t, r.URL.String(), url)
355
356                 fireEvent(t, r.Body)
357
358                 w.Header().Add("Content-Type", "application/json")
359                 w.WriteHeader(status)
360                 b, _ := json.Marshal(respData)
361                 w.Write(b)
362         }))
363         ts.Listener.Close()
364         ts.Listener = l
365
366         ts.Start()
367
368         return ts
369 }
370
371 func waitForEvent() string {
372         receivedAlert := <-eventChan
373         return receivedAlert
374 }
375
376 func fireEvent(t *testing.T, body io.ReadCloser) {
377         reqBody, err := ioutil.ReadAll(body)
378         assert.Nil(t, err, "ioutil.ReadAll failed")
379         assert.NotNil(t, reqBody, "ioutil.ReadAll failed")
380
381         eventChan <- fmt.Sprintf("%s", reqBody)
382 }
383
384 func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
385         rr := httptest.NewRecorder()
386
387         handleR.ServeHTTP(rr, req)
388
389         return rr
390 }
391
392 func checkResponseCode(t *testing.T, expected, actual int) bool {
393         if expected != actual {
394                 t.Errorf("Expected response code %d. Got %d\n", expected, actual)
395                 return false
396         }
397         return true
398 }