// Temp alarm constants & definitions
const (
- PERFORMANCE_TEST_ALARM_1 int = 1001
- PERFORMANCE_TEST_ALARM_2 int = 1002
- PERFORMANCE_TEST_ALARM_3 int = 1003
- PERFORMANCE_TEST_ALARM_4 int = 1004
- PERFORMANCE_TEST_ALARM_5 int = 1005
- PERFORMANCE_TEST_ALARM_6 int = 1006
- PERFORMANCE_TEST_ALARM_7 int = 1007
- PERFORMANCE_TEST_ALARM_8 int = 1008
- PERFORMANCE_TEST_ALARM_9 int = 1009
- PERFORMANCE_TEST_ALARM_10 int = 1010
- PERFORMANCE_TEST_ALARM_11 int = 1011
- PERFORMANCE_TEST_ALARM_12 int = 1012
- PERFORMANCE_TEST_ALARM_13 int = 1013
- PERFORMANCE_TEST_ALARM_14 int = 1014
- PERFORMANCE_TEST_ALARM_15 int = 1015
- PERFORMANCE_TEST_ALARM_16 int = 1016
- PERFORMANCE_TEST_ALARM_17 int = 1017
- PERFORMANCE_TEST_ALARM_18 int = 1018
- PERFORMANCE_TEST_ALARM_19 int = 1019
- PERFORMANCE_TEST_ALARM_20 int = 1020
- PERFORMANCE_TEST_ALARM_21 int = 1021
- PERFORMANCE_TEST_ALARM_22 int = 1022
- PERFORMANCE_TEST_ALARM_23 int = 1023
- PERFORMANCE_TEST_ALARM_24 int = 1024
- PERFORMANCE_TEST_ALARM_25 int = 1025
- PERFORMANCE_TEST_ALARM_26 int = 1026
- PERFORMANCE_TEST_ALARM_27 int = 1027
- PERFORMANCE_TEST_ALARM_28 int = 1028
- PERFORMANCE_TEST_ALARM_29 int = 1029
- PERFORMANCE_TEST_ALARM_30 int = 1030
- PERFORMANCE_TEST_ALARM_31 int = 1031
- PERFORMANCE_TEST_ALARM_32 int = 1032
- PERFORMANCE_TEST_ALARM_33 int = 1033
- PERFORMANCE_TEST_ALARM_34 int = 1034
- PERFORMANCE_TEST_ALARM_35 int = 1035
- PERFORMANCE_TEST_ALARM_36 int = 1036
- PERFORMANCE_TEST_ALARM_37 int = 1037
- PERFORMANCE_TEST_ALARM_38 int = 1038
- PERFORMANCE_TEST_ALARM_39 int = 1039
- PERFORMANCE_TEST_ALARM_40 int = 1040
- PERFORMANCE_TEST_ALARM_41 int = 1041
- PERFORMANCE_TEST_ALARM_42 int = 1042
- PERFORMANCE_TEST_ALARM_43 int = 1043
- PERFORMANCE_TEST_ALARM_44 int = 1044
- PERFORMANCE_TEST_ALARM_45 int = 1045
- PERFORMANCE_TEST_ALARM_46 int = 1046
- PERFORMANCE_TEST_ALARM_47 int = 1047
- PERFORMANCE_TEST_ALARM_48 int = 1048
- PERFORMANCE_TEST_ALARM_49 int = 1049
- PERFORMANCE_TEST_ALARM_50 int = 1050
RIC_RT_DISTRIBUTION_FAILED int = 8004
TCP_CONNECTIVITY_LOST_TO_DBAAS int = 8005
E2_CONNECTIVITY_LOST_TO_GNODEB int = 8006
OperationInstructions string `json:"operationInstructions"`
RaiseDelay int `json:"raiseDelay"`
ClearDelay int `json:"clearDelay"`
+ TimeToLive int `json:"timeToLive"`
}
var RICAlarmDefinitions map[int]*AlarmDefinition
GO111MODULE=on GO_ENABLED=0 GOOS=linux
# setup version tag
-if [ -f container-tag.yaml ]
+if [ -f build/container-tag.yaml ]
then
- tag=$(grep "tag:" container-tag.yaml | awk '{print $2}')
+ tag=$(grep "tag:" build/container-tag.yaml | awk '{print $2}')
else
tag="no-tag-found"
fi
# By default this file is in the docker build directory,
# but the location can configured in the JJB template.
---
-tag: 0.5.7
+tag: 0.5.8
"eventType" : "processingError",
"operationInstructions" : "Not defined",
"raiseDelay" : 0,
- "clearDelay" : 0
+ "clearDelay" : 0,
+ "timeToLive" : 0
},
{
"alarmId" : 8005,
"eventType" : "communication",
"operationInstructions" : "Not defined",
"raiseDelay" : 0,
- "clearDelay" : 0
+ "clearDelay" : 0,
+ "timeToLive" : 0
},
{
"alarmId" : 8006,
"eventType" : "communication",
"operationInstructions" : "Not defined",
"raiseDelay" : 0,
- "clearDelay" : 0
+ "clearDelay" : 0,
+ "timeToLive" : 0
},
{
"alarmId" : 8007,
"eventType" : "communication",
"operationInstructions" : "Not defined",
"raiseDelay" : 0,
- "clearDelay" : 0
+ "clearDelay" : 0,
+ "timeToLive" : 0
},
{
"alarmId" : 8008,
"eventType" : "equipment",
"operationInstructions" : "Clear alarms or raise threshold",
"raiseDelay" : 0,
- "clearDelay" : 0
+ "clearDelay" : 0,
+ "timeToLive" : 0
},
{
"alarmId" : 8009,
"eventType" : "equipment",
"operationInstructions" : "Clear alarms or raise threshold",
"raiseDelay" : 0,
- "clearDelay" : 0
+ "clearDelay" : 0,
+ "timeToLive" : 0
}
]
}
"github.com/spf13/viper"
)
+func (a *AlarmManager) ClearExpiredAlarms(m AlarmNotification, idx int, mLocked bool) bool {
+ d, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]
+ if !ok || d.TimeToLive == 0 {
+ return false
+ }
+
+ elapsed := (time.Now().UnixNano() - m.AlarmTime) / 1e9
+ if int(elapsed) >= d.TimeToLive {
+ app.Logger.Info("Alarm (sp=%d id=%d) with TTL=%d expired, clearing ...", m.Alarm.SpecificProblem, m.AlarmId, d.TimeToLive)
+
+ m.AlarmAction = alarm.AlarmActionClear
+ m.AlarmTime = time.Now().UnixNano()
+
+ if !mLocked { // For testing purpose
+ a.mutex.Lock()
+ }
+ a.ProcessClearAlarm(&m, d, idx)
+ return true
+ }
+ return false
+}
+
+func (a *AlarmManager) StartTTLTimer(interval int) {
+ tick := time.Tick(time.Duration(interval) * time.Second)
+ for range tick {
+ a.mutex.Lock()
+ for idx, m := range a.activeAlarms {
+ if a.ClearExpiredAlarms(m, idx, true) {
+ a.mutex.Lock() // ClearExpiredAlarms unlocks the mutex, so re-lock here
+ continue
+ }
+ }
+ a.mutex.Unlock()
+ }
+}
+
func (a *AlarmManager) StartAlertTimer() {
tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
for range tick {
thresholdMessage := alarm.AlarmMessage{
Alarm: thresholdAlarm,
AlarmAction: alarm.AlarmActionRaise,
- AlarmTime: (time.Now().UnixNano()),
+ AlarmTime: time.Now().UnixNano(),
}
alarmDef := alarm.RICAlarmDefinitions[sp]
alarmId := a.GenerateAlarmId()
ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
+ ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive
alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
}
}
alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
+
copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
+
wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
if err != nil {
app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
}
}
-func (a *AlarmManager) Run(sdlcheck bool) {
+func (a *AlarmManager) Run(sdlcheck bool, ttlInterval int) {
app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
app.Resource.InjectStatusCb(a.StatusCB)
alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
a.ReadAlarmDefinitionFromJson()
- app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
- app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
- app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
- app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
- app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST")
- app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET")
- app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST")
- app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE")
- app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET")
- app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET")
+ a.InjectRoutes()
// Start background timer for re-raising alerts
go a.StartAlertTimer()
+ go a.StartTTLTimer(ttlInterval)
+
a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
a.ReadAlarmInfoFromPersistentVolume()
// Main function
func main() {
- NewAlarmManager("", 0, true).Run(true)
+ NewAlarmManager("", 0, true).Run(true, 10)
}
func TestMain(M *testing.M) {
alarmManager = NewAlarmManager("localhost:9093", 500, false)
alarmManager.alertInterval = 20000
- go alarmManager.Run(false)
+ go alarmManager.Run(false, 5)
time.Sleep(time.Duration(10) * time.Second)
// Wait until RMR is up-and-running
assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
}
+func TestClearExpiredAlarms(t *testing.T) {
+ xapp.Logger.Info("TestClearExpiredAlarms")
+
+ a := alarm.AlarmMessage{
+ Alarm: alarmer.NewAlarm(8007, alarm.SeverityWarning, "threshold", ""),
+ AlarmAction: alarm.AlarmActionRaise,
+ AlarmTime: time.Now().UnixNano(),
+ }
+ d := alarm.RICAlarmDefinitions[8007]
+ n := AlarmNotification{a, *d}
+ alarmManager.activeAlarms = make([]AlarmNotification, 0)
+ alarmManager.UpdateActiveAlarmList(&n)
+
+ // Unknown SP
+ a.Alarm.SpecificProblem = 1234
+ assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+
+ // TTL is 0
+ d.TimeToLive = 0
+ assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+
+ // TTL not expired
+ a.Alarm.SpecificProblem = 8007
+ d.TimeToLive = 2
+ assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+
+ // TTL expired, alarm should be cleared
+ time.Sleep(time.Duration(3) * time.Second)
+ assert.Equal(t, len(alarmManager.activeAlarms), 1)
+ assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+ assert.Equal(t, len(alarmManager.activeAlarms), 0)
+}
+
func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
receivedAlert := waitForEvent()
"time"
)
+func (a *AlarmManager) InjectRoutes() {
+ app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
+ app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
+ app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
+ app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
+ app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST")
+ app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET")
+ app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST")
+ app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE")
+ app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET")
+ app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET")
+}
+
func (a *AlarmManager) respondWithError(w http.ResponseWriter, code int, message string) {
a.respondWithJSON(w, code, map[string]string{"error": message})
}
ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
+ ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive
alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
app.Logger.Debug("POST - alarm definition added for alarm id %v", alarmDefinition.AlarmId)
}