2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
32 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
33 app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
34 clientruntime "github.com/go-openapi/runtime/client"
35 "github.com/go-openapi/strfmt"
36 "github.com/prometheus/alertmanager/api/v2/client"
37 "github.com/prometheus/alertmanager/api/v2/client/alert"
38 "github.com/prometheus/alertmanager/api/v2/models"
39 "github.com/spf13/viper"
42 func (a *AlarmManager) ClearExpiredAlarms(m AlarmNotification, idx int, mLocked bool) bool {
43 d, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]
44 if !ok || d.TimeToLive == 0 {
48 elapsed := (time.Now().UnixNano() - m.AlarmTime) / 1e9
49 if int(elapsed) >= d.TimeToLive {
50 app.Logger.Info("Alarm (sp=%d id=%d) with TTL=%d expired, clearing ...", m.Alarm.SpecificProblem, m.AlarmId, d.TimeToLive)
52 m.AlarmAction = alarm.AlarmActionClear
53 m.AlarmTime = time.Now().UnixNano()
55 if !mLocked { // For testing purpose
58 a.ProcessClearAlarm(&m, d, idx)
64 func (a *AlarmManager) StartTTLTimer(interval int) {
65 tick := time.Tick(time.Duration(interval) * time.Second)
68 for idx, m := range a.activeAlarms {
69 if a.ClearExpiredAlarms(m, idx, true) {
70 a.mutex.Lock() // ClearExpiredAlarms unlocks the mutex, so re-lock here
78 func (a *AlarmManager) StartAlertTimer() {
79 tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
82 for _, m := range a.activeAlarms {
83 app.Logger.Info("Re-raising alarm: %v", m)
84 a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
90 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
91 app.Logger.Info("Message received!")
93 defer app.Rmr.Free(rp.Mbuf)
95 case alarm.RIC_ALARM_UPDATE:
98 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
104 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
105 var m alarm.AlarmMessage
106 app.Logger.Info("Received JSON: %s", rp.Payload)
107 if err := json.Unmarshal(rp.Payload, &m); err != nil {
108 app.Logger.Error("json.Unmarshal failed: %v", err)
111 app.Logger.Info("newAlarm: %v", m)
113 return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
116 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
118 alarmDef := &alarm.AlarmDefinition{}
120 if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
121 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
126 idx, found := a.IsMatchFound(m.Alarm)
127 // Suppress duplicate alarms
128 if found && m.AlarmAction == alarm.AlarmActionRaise {
129 app.Logger.Info("Duplicate alarm found, suppressing ...")
130 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
131 // Duplicate with same severity found
135 // Remove duplicate with different severity
136 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
140 // Clear alarm if found from active alarm list
141 if found && m.AlarmAction == alarm.AlarmActionClear {
142 return a.ProcessClearAlarm(m, alarmDef, idx)
145 // New alarm -> update active alarms and post to Alert Manager
146 if m.AlarmAction == alarm.AlarmActionRaise {
147 return a.ProcessRaiseAlarm(m, alarmDef)
154 func (a *AlarmManager) ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) (*alert.PostAlertsOK, error) {
155 app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
157 // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
158 m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
159 a.UpdateAlarmFields(a.GenerateAlarmId(), m)
160 a.UpdateActiveAlarmList(m)
163 if alarmDef.RaiseDelay > 0 {
164 timerDelay(alarmDef.RaiseDelay)
166 // Alarm may have been deleted from active alarms table during delay or table index may have changed
167 idx, found := a.IsMatchFound(m.Alarm)
169 // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
170 a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
171 app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
174 app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
180 m.AlarmDefinition.RaiseDelay = 0
181 a.UpdateAlarmHistoryList(m)
182 a.WriteAlarmInfoToPersistentVolume()
184 // Send alarm notification to NOMA, if enabled
185 if app.Config.GetBool("controls.noma.enabled") {
186 return a.PostAlarm(m)
188 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
191 func (a *AlarmManager) ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) (*alert.PostAlertsOK, error) {
192 app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
193 if alarmDef.ClearDelay > 0 {
195 timerDelay(alarmDef.ClearDelay)
196 app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
198 // Another alarm clear may have happened during delay and active alarms table index changed
200 idx, found = a.IsMatchFound(m.Alarm)
206 a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
207 a.alarmHistory = append(a.alarmHistory, *m)
208 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
209 if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
210 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
211 a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
214 if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
215 a.exceededActiveAlarmOn = false
218 if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
219 a.exceededAlarmHistoryOn = false
221 a.WriteAlarmInfoToPersistentVolume()
224 if a.postClear && app.Config.GetBool("controls.noma.enabled") {
225 m.PerceivedSeverity = alarm.SeverityCleared
226 return a.PostAlarm(m)
231 func timerDelay(delay int) {
232 timer := time.NewTimer(time.Duration(delay) * time.Second)
236 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
237 for i, m := range a.activeAlarms {
238 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
239 m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
246 func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
247 app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
248 copy(alarms[i:], alarms[i+1:])
249 return alarms[:len(alarms)-1]
252 func (a *AlarmManager) GenerateAlarmId() int {
253 a.uniqueAlarmId++ // @todo: generate a unique ID
254 return a.uniqueAlarmId
257 func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
258 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
259 newAlarm.AlarmId = alarmId
260 newAlarm.AlarmText = alarmDef.AlarmText
261 newAlarm.EventType = alarmDef.EventType
264 func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
265 thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
266 thresholdMessage := alarm.AlarmMessage{
267 Alarm: thresholdAlarm,
268 AlarmAction: alarm.AlarmActionRaise,
269 AlarmTime: time.Now().UnixNano(),
271 alarmDef := alarm.RICAlarmDefinitions[sp]
272 alarmId := a.GenerateAlarmId()
273 alarmDef.AlarmId = alarmId
274 a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, *alarmDef})
275 a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, *alarmDef})
280 func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
281 /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
282 The attempt to raise the alarm next time will be suppressed when found as duplicate. */
283 if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
284 app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
285 a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
288 // @todo: For now just keep the active alarms in-memory. Use SDL later for persistence
289 a.activeAlarms = append(a.activeAlarms, *newAlarm)
292 func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
293 /* If maximum number of events in alarm history is reached, an error log writing is made,
294 and new alarm indicating the problem is raised. The attempt to add new event time will
297 if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
298 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
299 a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
302 // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
303 a.alarmHistory = append(a.alarmHistory, *newAlarm)
306 func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
307 result, err := json.Marshal(m)
309 app.Logger.Info("json.Marshal failed: %v", err)
313 fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
314 app.Logger.Info("Posting alarm to '%s'", fullUrl)
316 resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
317 if err != nil || resp == nil {
318 app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
324 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
325 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
326 amLabels := models.LabelSet{
327 "status": string(status),
328 "alertname": alarmDef.AlarmText,
329 "severity": string(newAlarm.PerceivedSeverity),
330 "service": fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
331 "system_name": "RIC",
333 amAnnotations := models.LabelSet{
334 "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
335 "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
336 "event_type": alarmDef.EventType,
337 "identifying_info": newAlarm.IdentifyingInfo,
338 "additional_info": newAlarm.AdditionalInfo,
339 "description": fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
340 "instructions": alarmDef.OperationInstructions,
341 "timestamp": fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
344 return amLabels, amAnnotations
347 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
348 cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
349 return client.New(cr, strfmt.Default)
352 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
353 pa := &models.PostableAlert{
355 GeneratorURL: strfmt.URI(""),
358 Annotations: amAnnotations,
360 alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
362 app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
363 ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
365 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
370 func (a *AlarmManager) StatusCB() bool {
372 app.Logger.Info("RMR not ready yet!")
378 func (a *AlarmManager) ConfigChangeCB(configparam string) {
380 a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
381 a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
382 a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
383 a.amHost = viper.GetString("controls.promAlertManager.address")
385 app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
386 app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
387 app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
388 app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
393 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
395 filename := os.Getenv("DEF_FILE")
396 file, err := ioutil.ReadFile(filename)
398 data := RicAlarmDefinitions{}
399 err = json.Unmarshal([]byte(file), &data)
401 for _, alarmDefinition := range data.AlarmDefinitions {
402 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
404 app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
406 app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm %v", alarmDefinition.AlarmId)
407 ricAlarmDefintion := new(alarm.AlarmDefinition)
408 ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
409 ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
410 ricAlarmDefintion.EventType = alarmDefinition.EventType
411 ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
412 ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
413 ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
414 ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive
415 alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
419 app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
422 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
426 func (a *AlarmManager) ReadAlarmInfoFromPersistentVolume() {
427 var alarmpersistentinfo AlarmPersistentInfo
428 byteValue, rerr := ioutil.ReadFile(a.alarmInfoPvFile)
430 app.Logger.Error("ararminfo.json file read error %v", rerr)
432 err := json.Unmarshal(byteValue, &alarmpersistentinfo)
434 app.Logger.Error("alarmpersistentinfo json unmarshal error %v", err)
436 a.uniqueAlarmId = alarmpersistentinfo.UniqueAlarmId
437 a.activeAlarms = make([]AlarmNotification, len(alarmpersistentinfo.ActiveAlarms))
438 a.alarmHistory = make([]AlarmNotification, len(alarmpersistentinfo.AlarmHistory))
439 copy(a.activeAlarms, alarmpersistentinfo.ActiveAlarms)
440 copy(a.alarmHistory, alarmpersistentinfo.AlarmHistory)
445 func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() {
446 var alarmpersistentinfo AlarmPersistentInfo
447 alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
448 alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
449 alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
451 copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
452 copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
454 wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
456 app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
458 werr := ioutil.WriteFile(a.alarmInfoPvFile, wdata, 0777)
460 app.Logger.Error("alarminfo.json file write error %v", werr)
465 func (a *AlarmManager) Run(sdlcheck bool, ttlInterval int) {
466 app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
467 app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
468 app.Resource.InjectStatusCb(a.StatusCB)
469 app.AddConfigChangeListener(a.ConfigChangeCB)
471 alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
472 a.ReadAlarmDefinitionFromJson()
476 // Start background timer for re-raising alerts
477 go a.StartAlertTimer()
478 go a.StartTTLTimer(ttlInterval)
480 a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
482 a.ReadAlarmInfoFromPersistentVolume()
484 app.RunWithParams(a, sdlcheck)
487 func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
488 if alertInterval == 0 {
489 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
493 amHost = viper.GetString("controls.promAlertManager.address")
496 return &AlarmManager{
498 postClear: clearAlarm,
500 amBaseUrl: app.Config.GetString("controls.promAlertManager.baseUrl"),
501 amSchemes: []string{app.Config.GetString("controls.promAlertManager.schemes")},
502 alertInterval: alertInterval,
503 activeAlarms: make([]AlarmNotification, 0),
504 alarmHistory: make([]AlarmNotification, 0),
506 maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
507 maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
508 exceededActiveAlarmOn: false,
509 exceededAlarmHistoryOn: false,
510 alarmInfoPvFile: app.Config.GetString("controls.alarmInfoPvFile"),
516 NewAlarmManager("", 0, true).Run(true, 10)