From: Mohamed Abukar Date: Fri, 30 Oct 2020 14:19:38 +0000 (+0200) Subject: More code refactoring X-Git-Tag: alarm/v0.5.0~9 X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=2336a84ee4d6f4596c0c4556040396022f8f08a2;p=ric-plt%2Falarm-go.git More code refactoring Change-Id: I3d407763112c7e809f501e7aa69278af40a18b15 Signed-off-by: Mohamed Abukar --- diff --git a/cli/alarm-cli.go b/cli/alarm-cli.go index 1b630cd..775b806 100755 --- a/cli/alarm-cli.go +++ b/cli/alarm-cli.go @@ -70,7 +70,21 @@ func main() { SetVersion("1.0.0"). SetDescription("This CLI tool provides management interface to SEP alarm system") - // Get active alarms + registerActiveCmd(alarmManagerHost) + registerHistoryCmd(alarmManagerHost) + registerRaiseCmd(alarmManagerHost) + registerClearCmd(alarmManagerHost) + registerDefineCmd(alarmManagerHost) + registerUndefineCmd(alarmManagerHost) + registerConfigureCmd(alarmManagerHost) + registerPerfCmd(alarmManagerHost) + registerAlertCmd(alertManagerHost) + + // parse command-line arguments + commando.Parse(nil) +} + +func registerActiveCmd(alarmManagerHost string) { commando. Register("active"). SetShortDescription("Displays the SEP active alarms"). @@ -80,7 +94,9 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { displayAlarms(getAlarms(flags, "active"), false) }) +} +func registerHistoryCmd(alarmManagerHost string) { // Get alarm history commando. Register("history"). @@ -91,7 +107,9 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { displayAlarms(getAlarms(flags, "history"), true) }) +} +func registerRaiseCmd(alarmManagerHost string) { // Raise an alarm commando. Register("raise"). @@ -109,6 +127,9 @@ func main() { postAlarm(flags, readAlarmParams(flags, false), alarm.AlarmActionRaise, nil) }) +} + +func registerClearCmd(alarmManagerHost string) { // Clear an alarm commando. Register("clear"). @@ -124,6 +145,9 @@ func main() { postAlarm(flags, readAlarmParams(flags, true), alarm.AlarmActionClear, nil) }) +} + +func registerConfigureCmd(alarmManagerHost string) { // Configure an alarm manager commando. Register("configure"). @@ -135,6 +159,9 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { postAlarmConfig(flags) }) +} + +func registerDefineCmd(alarmManagerHost string) { // Create alarm definition commando. Register("define"). @@ -150,7 +177,10 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { postAlarmDefinition(flags) }) - // Delete alarm definition +} + +func registerUndefineCmd(alarmManagerHost string) { + // Delete alarm definition commando. Register("undefine"). SetShortDescription("Define alarm with given parameters"). @@ -160,7 +190,10 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { deleteAlarmDefinition(flags) }) - // Conduct performance test for alarm-go +} + +func registerPerfCmd(alarmManagerHost string) { + // Conduct performance test for alarm-go commando. Register("perf"). SetShortDescription("Conduct performance test with given parameters"). @@ -174,10 +207,12 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { conductperformancetest(flags) }) +} +func registerAlertCmd(alertManagerHost string) { // Get alerts from Prometheus Alert Manager commando. - Register("gapam"). + Register("alerts"). SetShortDescription("Get alerts from Prometheus Alert Manager"). AddFlag("active", "Active alerts in Prometheus Alert Manager", commando.Bool, true). AddFlag("inhibited", "Inhibited alerts in Prometheus Alert Manager", commando.Bool, true). @@ -188,9 +223,6 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { displayAlerts(flags) }) - - // parse command-line arguments - commando.Parse(nil) } func readAlarmParams(flags map[string]commando.FlagValue, clear bool) (a alarm.Alarm) { @@ -540,7 +572,7 @@ func readPerfAlarmDefinitionFromJson() error { } else { fmt.Printf("ReadPerfAlarmDefinitionFromJson: ioutil.ReadFile failed with error: %v, filename: %s\n", err, filename) fmt.Printf("ReadPerfAlarmDefinitionFromJson: current directory: %s\n", getCurrentDirectory()) - + return err } return nil @@ -715,4 +747,4 @@ func getCurrentDirectory() string { fmt.Println(err) } return dir -} \ No newline at end of file +} diff --git a/manager/cmd/manager.go b/manager/cmd/manager.go index 0036679..789093c 100755 --- a/manager/cmd/manager.go +++ b/manager/cmd/manager.go @@ -102,53 +102,28 @@ func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, } // Clear alarm if found from active alarm list - if m.AlarmAction == alarm.AlarmActionClear { - if !found { - app.Logger.Info("No matching active alarm found, suppressing ...") - a.mutex.Unlock() - return nil, nil - } - - if a.ProcessClearAlarm(m, alarmDef, idx) == false { - return nil, nil - } - - a.mutex.Unlock() - if !a.postClear { - app.Logger.Info("Sending clear notification disabled!") - return nil, nil - } - - // Send alarm notification to NOMA, if enabled - if app.Config.GetBool("controls.noma.enabled") { - m.PerceivedSeverity = alarm.SeverityCleared - return a.PostAlarm(m) - } + if found && m.AlarmAction == alarm.AlarmActionClear { + return a.ProcessClearAlarm(m, alarmDef, idx) } // New alarm -> update active alarms and post to Alert Manager if m.AlarmAction == alarm.AlarmActionRaise { - if a.ProcessRaiseAlarm(m, alarmDef) == false { - return nil, nil - } - // Send alarm notification to NOMA, if enabled - if app.Config.GetBool("controls.noma.enabled") { - return a.PostAlarm(m) - } - return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime)) + return a.ProcessRaiseAlarm(m, alarmDef) } a.mutex.Unlock() return nil, nil } -func (a *AlarmManager)ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) bool { +func (a *AlarmManager) ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) (*alert.PostAlertsOK, error) { app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m) + // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay a.UpdateAlarmFields(a.GenerateAlarmId(), m) a.UpdateActiveAlarmList(m) a.mutex.Unlock() + if alarmDef.RaiseDelay > 0 { timerDelay(alarmDef.RaiseDelay) a.mutex.Lock() @@ -162,16 +137,22 @@ func (a *AlarmManager)ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.Al } else { app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m) a.mutex.Unlock() - return false + return nil, nil } } + m.AlarmDefinition.RaiseDelay = 0 a.UpdateAlarmHistoryList(m) - a.WriteAlarmInfoToPersistentVolume() - return true + a.WriteAlarmInfoToPersistentVolume() + + // Send alarm notification to NOMA, if enabled + if app.Config.GetBool("controls.noma.enabled") { + return a.PostAlarm(m) + } + return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime)) } -func (a *AlarmManager)ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) bool { +func (a *AlarmManager) ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) (*alert.PostAlertsOK, error) { app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m) if alarmDef.ClearDelay > 0 { a.mutex.Unlock() @@ -182,9 +163,8 @@ func (a *AlarmManager)ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.Al var found bool idx, found = a.IsMatchFound(m.Alarm) if !found { - app.Logger.Debug("Alarm not anymore in the active alarms table. AlarmNotification = %v", *m) a.mutex.Unlock() - return false + return nil, nil } } a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m) @@ -202,8 +182,14 @@ func (a *AlarmManager)ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.Al if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD { a.exceededAlarmHistoryOn = false } - a.WriteAlarmInfoToPersistentVolume() - return true + a.WriteAlarmInfoToPersistentVolume() + + a.mutex.Unlock() + if a.postClear && app.Config.GetBool("controls.noma.enabled") { + m.PerceivedSeverity = alarm.SeverityCleared + return a.PostAlarm(m) + } + return nil, nil } func timerDelay(delay int) { diff --git a/manager/cmd/manager_test.go b/manager/cmd/manager_test.go index b75cae5..c50084b 100755 --- a/manager/cmd/manager_test.go +++ b/manager/cmd/manager_test.go @@ -392,17 +392,17 @@ func TestDelayedAlarmRaiseAndClear(t *testing.T) { ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) defer ts.Close() - // Raise alarm. Posting alert and updating alarm history should be delayed + // Raise alarm. Posting alert and updating alarm history should be delayed a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(a), "raise failed") - VerifyAlarm(t, a, activeAlarmsBeforeTest + 1) + VerifyAlarm(t, a, activeAlarmsBeforeTest+1) // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed assert.Nil(t, alarmer.Clear(a), "clear failed") time.Sleep(time.Duration(2) * time.Second) assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) - assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 2) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2) } func TestDelayedAlarmRaiseAndClear2(t *testing.T) { @@ -417,11 +417,11 @@ func TestDelayedAlarmRaiseAndClear2(t *testing.T) { // Raise two alarms. The first should be delayed a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(a), "raise failed") - VerifyAlarm(t, a, activeAlarmsBeforeTest + 1) + VerifyAlarm(t, a, activeAlarmsBeforeTest+1) b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(b), "raise failed") - VerifyAlarm(t, b, activeAlarmsBeforeTest + 2) + VerifyAlarm(t, b, activeAlarmsBeforeTest+2) // Clear two alarms. The first should be delayed. Check the alarms are removed assert.Nil(t, alarmer.Clear(a), "clear failed") @@ -429,7 +429,7 @@ func TestDelayedAlarmRaiseAndClear2(t *testing.T) { time.Sleep(time.Duration(2) * time.Second) assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) - assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4) } func TestDelayedAlarmRaiseAndClear3(t *testing.T) { @@ -475,11 +475,11 @@ func TestDelayedAlarmRaiseAndClear3(t *testing.T) { // Raise two alarms. The first should be delayed a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(a), "raise failed") - VerifyAlarm(t, a, activeAlarmsBeforeTest + 1) + VerifyAlarm(t, a, activeAlarmsBeforeTest+1) b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(b), "raise failed") - VerifyAlarm(t, b, activeAlarmsBeforeTest + 2) + VerifyAlarm(t, b, activeAlarmsBeforeTest+2) // Clear two alarms. The first should be delayed. Check the alarms are removed assert.Nil(t, alarmer.Clear(a), "clear failed") @@ -487,7 +487,7 @@ func TestDelayedAlarmRaiseAndClear3(t *testing.T) { time.Sleep(time.Duration(2) * time.Second) assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) - assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4) } func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {