From 3649fae7a06ad3ad099d0aa4e68f7ca3a2ae5a87 Mon Sep 17 00:00:00 2001 From: Mohamed Abukar Date: Fri, 30 Oct 2020 23:51:39 +0200 Subject: [PATCH] Support for time-to-live Change-Id: I475fe24471aa40c2612b882db12c4d3c7cfef5f4 Signed-off-by: Mohamed Abukar --- alarm/types.go | 51 +--------------------------------- build/build_ubuntu.sh | 4 +-- build/container-tag.yaml | 2 +- definitions/alarm-definition.json | 18 ++++++++---- manager/cmd/manager.go | 58 ++++++++++++++++++++++++++++++--------- manager/cmd/manager_test.go | 35 ++++++++++++++++++++++- manager/cmd/restapi.go | 14 ++++++++++ 7 files changed, 109 insertions(+), 73 deletions(-) diff --git a/alarm/types.go b/alarm/types.go index 200d435..14a179c 100755 --- a/alarm/types.go +++ b/alarm/types.go @@ -89,56 +89,6 @@ const ( // Temp alarm constants & definitions const ( - PERFORMANCE_TEST_ALARM_1 int = 1001 - PERFORMANCE_TEST_ALARM_2 int = 1002 - PERFORMANCE_TEST_ALARM_3 int = 1003 - PERFORMANCE_TEST_ALARM_4 int = 1004 - PERFORMANCE_TEST_ALARM_5 int = 1005 - PERFORMANCE_TEST_ALARM_6 int = 1006 - PERFORMANCE_TEST_ALARM_7 int = 1007 - PERFORMANCE_TEST_ALARM_8 int = 1008 - PERFORMANCE_TEST_ALARM_9 int = 1009 - PERFORMANCE_TEST_ALARM_10 int = 1010 - PERFORMANCE_TEST_ALARM_11 int = 1011 - PERFORMANCE_TEST_ALARM_12 int = 1012 - PERFORMANCE_TEST_ALARM_13 int = 1013 - PERFORMANCE_TEST_ALARM_14 int = 1014 - PERFORMANCE_TEST_ALARM_15 int = 1015 - PERFORMANCE_TEST_ALARM_16 int = 1016 - PERFORMANCE_TEST_ALARM_17 int = 1017 - PERFORMANCE_TEST_ALARM_18 int = 1018 - PERFORMANCE_TEST_ALARM_19 int = 1019 - PERFORMANCE_TEST_ALARM_20 int = 1020 - PERFORMANCE_TEST_ALARM_21 int = 1021 - PERFORMANCE_TEST_ALARM_22 int = 1022 - PERFORMANCE_TEST_ALARM_23 int = 1023 - PERFORMANCE_TEST_ALARM_24 int = 1024 - PERFORMANCE_TEST_ALARM_25 int = 1025 - PERFORMANCE_TEST_ALARM_26 int = 1026 - PERFORMANCE_TEST_ALARM_27 int = 1027 - PERFORMANCE_TEST_ALARM_28 int = 1028 - PERFORMANCE_TEST_ALARM_29 int = 1029 - PERFORMANCE_TEST_ALARM_30 int = 1030 - PERFORMANCE_TEST_ALARM_31 int = 1031 - PERFORMANCE_TEST_ALARM_32 int = 1032 - PERFORMANCE_TEST_ALARM_33 int = 1033 - PERFORMANCE_TEST_ALARM_34 int = 1034 - PERFORMANCE_TEST_ALARM_35 int = 1035 - PERFORMANCE_TEST_ALARM_36 int = 1036 - PERFORMANCE_TEST_ALARM_37 int = 1037 - PERFORMANCE_TEST_ALARM_38 int = 1038 - PERFORMANCE_TEST_ALARM_39 int = 1039 - PERFORMANCE_TEST_ALARM_40 int = 1040 - PERFORMANCE_TEST_ALARM_41 int = 1041 - PERFORMANCE_TEST_ALARM_42 int = 1042 - PERFORMANCE_TEST_ALARM_43 int = 1043 - PERFORMANCE_TEST_ALARM_44 int = 1044 - PERFORMANCE_TEST_ALARM_45 int = 1045 - PERFORMANCE_TEST_ALARM_46 int = 1046 - PERFORMANCE_TEST_ALARM_47 int = 1047 - PERFORMANCE_TEST_ALARM_48 int = 1048 - PERFORMANCE_TEST_ALARM_49 int = 1049 - PERFORMANCE_TEST_ALARM_50 int = 1050 RIC_RT_DISTRIBUTION_FAILED int = 8004 TCP_CONNECTIVITY_LOST_TO_DBAAS int = 8005 E2_CONNECTIVITY_LOST_TO_GNODEB int = 8006 @@ -154,6 +104,7 @@ type AlarmDefinition struct { OperationInstructions string `json:"operationInstructions"` RaiseDelay int `json:"raiseDelay"` ClearDelay int `json:"clearDelay"` + TimeToLive int `json:"timeToLive"` } var RICAlarmDefinitions map[int]*AlarmDefinition diff --git a/build/build_ubuntu.sh b/build/build_ubuntu.sh index 26bc0e3..28f0a6f 100755 --- a/build/build_ubuntu.sh +++ b/build/build_ubuntu.sh @@ -50,9 +50,9 @@ export PERF_OBJ_FILE=../testresources/perf-alarm-object.json GO111MODULE=on GO_ENABLED=0 GOOS=linux # setup version tag -if [ -f container-tag.yaml ] +if [ -f build/container-tag.yaml ] then - tag=$(grep "tag:" container-tag.yaml | awk '{print $2}') + tag=$(grep "tag:" build/container-tag.yaml | awk '{print $2}') else tag="no-tag-found" fi diff --git a/build/container-tag.yaml b/build/container-tag.yaml index 599041f..5349da6 100644 --- a/build/container-tag.yaml +++ b/build/container-tag.yaml @@ -2,4 +2,4 @@ # By default this file is in the docker build directory, # but the location can configured in the JJB template. --- -tag: 0.5.7 +tag: 0.5.8 diff --git a/definitions/alarm-definition.json b/definitions/alarm-definition.json index ce32693..f9c32d7 100755 --- a/definitions/alarm-definition.json +++ b/definitions/alarm-definition.json @@ -6,7 +6,8 @@ "eventType" : "processingError", "operationInstructions" : "Not defined", "raiseDelay" : 0, - "clearDelay" : 0 + "clearDelay" : 0, + "timeToLive" : 0 }, { "alarmId" : 8005, @@ -14,7 +15,8 @@ "eventType" : "communication", "operationInstructions" : "Not defined", "raiseDelay" : 0, - "clearDelay" : 0 + "clearDelay" : 0, + "timeToLive" : 0 }, { "alarmId" : 8006, @@ -22,7 +24,8 @@ "eventType" : "communication", "operationInstructions" : "Not defined", "raiseDelay" : 0, - "clearDelay" : 0 + "clearDelay" : 0, + "timeToLive" : 0 }, { "alarmId" : 8007, @@ -30,7 +33,8 @@ "eventType" : "communication", "operationInstructions" : "Not defined", "raiseDelay" : 0, - "clearDelay" : 0 + "clearDelay" : 0, + "timeToLive" : 0 }, { "alarmId" : 8008, @@ -38,7 +42,8 @@ "eventType" : "equipment", "operationInstructions" : "Clear alarms or raise threshold", "raiseDelay" : 0, - "clearDelay" : 0 + "clearDelay" : 0, + "timeToLive" : 0 }, { "alarmId" : 8009, @@ -46,7 +51,8 @@ "eventType" : "equipment", "operationInstructions" : "Clear alarms or raise threshold", "raiseDelay" : 0, - "clearDelay" : 0 + "clearDelay" : 0, + "timeToLive" : 0 } ] } diff --git a/manager/cmd/manager.go b/manager/cmd/manager.go index 789093c..10f5115 100755 --- a/manager/cmd/manager.go +++ b/manager/cmd/manager.go @@ -39,6 +39,42 @@ import ( "github.com/spf13/viper" ) +func (a *AlarmManager) ClearExpiredAlarms(m AlarmNotification, idx int, mLocked bool) bool { + d, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem] + if !ok || d.TimeToLive == 0 { + return false + } + + elapsed := (time.Now().UnixNano() - m.AlarmTime) / 1e9 + if int(elapsed) >= d.TimeToLive { + app.Logger.Info("Alarm (sp=%d id=%d) with TTL=%d expired, clearing ...", m.Alarm.SpecificProblem, m.AlarmId, d.TimeToLive) + + m.AlarmAction = alarm.AlarmActionClear + m.AlarmTime = time.Now().UnixNano() + + if !mLocked { // For testing purpose + a.mutex.Lock() + } + a.ProcessClearAlarm(&m, d, idx) + return true + } + return false +} + +func (a *AlarmManager) StartTTLTimer(interval int) { + tick := time.Tick(time.Duration(interval) * time.Second) + for range tick { + a.mutex.Lock() + for idx, m := range a.activeAlarms { + if a.ClearExpiredAlarms(m, idx, true) { + a.mutex.Lock() // ClearExpiredAlarms unlocks the mutex, so re-lock here + continue + } + } + a.mutex.Unlock() + } +} + func (a *AlarmManager) StartAlertTimer() { tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond) for range tick { @@ -230,7 +266,7 @@ func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool { thresholdMessage := alarm.AlarmMessage{ Alarm: thresholdAlarm, AlarmAction: alarm.AlarmActionRaise, - AlarmTime: (time.Now().UnixNano()), + AlarmTime: time.Now().UnixNano(), } alarmDef := alarm.RICAlarmDefinitions[sp] alarmId := a.GenerateAlarmId() @@ -375,6 +411,7 @@ func (a *AlarmManager) ReadAlarmDefinitionFromJson() { ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay + ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion } } @@ -410,8 +447,10 @@ func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() { alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms)) alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory)) + copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms) copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory) + wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ") if err != nil { app.Logger.Error("alarmpersistentinfo json marshal error %v", err) @@ -423,7 +462,7 @@ func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() { } } -func (a *AlarmManager) Run(sdlcheck bool) { +func (a *AlarmManager) Run(sdlcheck bool, ttlInterval int) { app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash)) app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true) app.Resource.InjectStatusCb(a.StatusCB) @@ -432,19 +471,12 @@ func (a *AlarmManager) Run(sdlcheck bool) { alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition) a.ReadAlarmDefinitionFromJson() - app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST") - app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE") - app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET") - app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET") - app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST") - app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET") - app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST") - app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE") - app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET") - app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET") + a.InjectRoutes() // Start background timer for re-raising alerts go a.StartAlertTimer() + go a.StartTTLTimer(ttlInterval) + a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER") a.ReadAlarmInfoFromPersistentVolume() @@ -481,5 +513,5 @@ func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmMa // Main function func main() { - NewAlarmManager("", 0, true).Run(true) + NewAlarmManager("", 0, true).Run(true, 10) } diff --git a/manager/cmd/manager_test.go b/manager/cmd/manager_test.go index c50084b..00df224 100755 --- a/manager/cmd/manager_test.go +++ b/manager/cmd/manager_test.go @@ -50,7 +50,7 @@ var eventChan chan string func TestMain(M *testing.M) { alarmManager = NewAlarmManager("localhost:9093", 500, false) alarmManager.alertInterval = 20000 - go alarmManager.Run(false) + go alarmManager.Run(false, 5) time.Sleep(time.Duration(10) * time.Second) // Wait until RMR is up-and-running @@ -490,6 +490,39 @@ func TestDelayedAlarmRaiseAndClear3(t *testing.T) { assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4) } +func TestClearExpiredAlarms(t *testing.T) { + xapp.Logger.Info("TestClearExpiredAlarms") + + a := alarm.AlarmMessage{ + Alarm: alarmer.NewAlarm(8007, alarm.SeverityWarning, "threshold", ""), + AlarmAction: alarm.AlarmActionRaise, + AlarmTime: time.Now().UnixNano(), + } + d := alarm.RICAlarmDefinitions[8007] + n := AlarmNotification{a, *d} + alarmManager.activeAlarms = make([]AlarmNotification, 0) + alarmManager.UpdateActiveAlarmList(&n) + + // Unknown SP + a.Alarm.SpecificProblem = 1234 + assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + + // TTL is 0 + d.TimeToLive = 0 + assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + + // TTL not expired + a.Alarm.SpecificProblem = 8007 + d.TimeToLive = 2 + assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + + // TTL expired, alarm should be cleared + time.Sleep(time.Duration(3) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), 1) + assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + assert.Equal(t, len(alarmManager.activeAlarms), 0) +} + func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string { receivedAlert := waitForEvent() diff --git a/manager/cmd/restapi.go b/manager/cmd/restapi.go index b8946df..b8062a0 100755 --- a/manager/cmd/restapi.go +++ b/manager/cmd/restapi.go @@ -30,6 +30,19 @@ import ( "time" ) +func (a *AlarmManager) InjectRoutes() { + app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST") + app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE") + app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET") + app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET") + app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST") + app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET") + app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST") + app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE") + app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET") + app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET") +} + func (a *AlarmManager) respondWithError(w http.ResponseWriter, code int, message string) { a.respondWithJSON(w, code, map[string]string{"error": message}) } @@ -97,6 +110,7 @@ func (a *AlarmManager) SetAlarmDefinition(w http.ResponseWriter, r *http.Request ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay + ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion app.Logger.Debug("POST - alarm definition added for alarm id %v", alarmDefinition.AlarmId) } -- 2.16.6