2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
32 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
33 app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
34 clientruntime "github.com/go-openapi/runtime/client"
35 "github.com/go-openapi/strfmt"
36 "github.com/prometheus/alertmanager/api/v2/client"
37 "github.com/prometheus/alertmanager/api/v2/client/alert"
38 "github.com/prometheus/alertmanager/api/v2/models"
39 "github.com/spf13/viper"
42 func (a *AlarmManager) StartAlertTimer() {
43 tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
46 for _, m := range a.activeAlarms {
47 app.Logger.Info("Re-raising alarm: %v", m)
48 a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
54 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
55 app.Logger.Info("Message received!")
57 defer app.Rmr.Free(rp.Mbuf)
59 case alarm.RIC_ALARM_UPDATE:
62 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
68 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
69 var m alarm.AlarmMessage
70 app.Logger.Info("Received JSON: %s", rp.Payload)
71 if err := json.Unmarshal(rp.Payload, &m); err != nil {
72 app.Logger.Error("json.Unmarshal failed: %v", err)
75 app.Logger.Info("newAlarm: %v", m)
77 return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
80 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
82 alarmDef := &alarm.AlarmDefinition{}
84 if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
85 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
90 idx, found := a.IsMatchFound(m.Alarm)
91 // Suppress duplicate alarms
92 if found && m.AlarmAction == alarm.AlarmActionRaise {
93 app.Logger.Info("Duplicate alarm found, suppressing ...")
94 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
95 // Duplicate with same severity found
99 // Remove duplicate with different severity
100 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
104 // Clear alarm if found from active alarm list
105 if m.AlarmAction == alarm.AlarmActionClear {
107 if a.ProcessClearAlarm(m, alarmDef, idx) == false {
113 // Send alarm notification to NOMA, if enabled
114 if app.Config.GetBool("controls.noma.enabled") {
115 m.PerceivedSeverity = alarm.SeverityCleared
116 return a.PostAlarm(m)
118 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved, m.AlarmTime))
121 app.Logger.Info("No matching active alarm found, suppressing ...")
126 // New alarm -> update active alarms and post to Alert Manager
127 if m.AlarmAction == alarm.AlarmActionRaise {
128 if a.ProcessRaiseAlarm(m, alarmDef) == false {
131 // Send alarm notification to NOMA, if enabled
132 if app.Config.GetBool("controls.noma.enabled") {
133 return a.PostAlarm(m)
135 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
142 func (a *AlarmManager)ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) bool {
143 app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
144 // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
145 m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
146 a.UpdateAlarmFields(a.GenerateAlarmId(), m)
147 a.UpdateActiveAlarmList(m)
149 if alarmDef.RaiseDelay > 0 {
150 timerDelay(alarmDef.RaiseDelay)
152 // Alarm may have been deleted from active alarms table during delay or table index may have changed
153 idx, found := a.IsMatchFound(m.Alarm)
155 // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
156 a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
157 app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
160 app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
165 m.AlarmDefinition.RaiseDelay = 0
166 a.UpdateAlarmHistoryList(m)
167 a.WriteAlarmInfoToPersistentVolume()
171 func (a *AlarmManager)ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) bool {
172 app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
173 if alarmDef.ClearDelay > 0 {
175 timerDelay(alarmDef.ClearDelay)
176 app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
178 // Another alarm clear may have happened during delay and active alarms table index changed
180 idx, found = a.IsMatchFound(m.Alarm)
182 app.Logger.Debug("Alarm not anymore in the active alarms table. AlarmNotification = %v", *m)
187 a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
188 a.alarmHistory = append(a.alarmHistory, *m)
189 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
190 if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
191 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
192 a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
195 if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
196 a.exceededActiveAlarmOn = false
199 if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
200 a.exceededAlarmHistoryOn = false
202 a.WriteAlarmInfoToPersistentVolume()
206 func timerDelay(delay int) {
207 timer := time.NewTimer(time.Duration(delay) * time.Second)
211 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
212 for i, m := range a.activeAlarms {
213 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
214 m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
221 func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
222 app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
223 copy(alarms[i:], alarms[i+1:])
224 return alarms[:len(alarms)-1]
227 func (a *AlarmManager) GenerateAlarmId() int {
228 a.uniqueAlarmId++ // @todo: generate a unique ID
229 return a.uniqueAlarmId
232 func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
233 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
234 newAlarm.AlarmId = alarmId
235 newAlarm.AlarmText = alarmDef.AlarmText
236 newAlarm.EventType = alarmDef.EventType
239 func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
240 thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
241 thresholdMessage := alarm.AlarmMessage{
242 Alarm: thresholdAlarm,
243 AlarmAction: alarm.AlarmActionRaise,
244 AlarmTime: (time.Now().UnixNano()),
246 alarmDef := alarm.RICAlarmDefinitions[sp]
247 alarmId := a.GenerateAlarmId()
248 alarmDef.AlarmId = alarmId
249 a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, *alarmDef})
250 a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, *alarmDef})
255 func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
256 /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
257 The attempt to raise the alarm next time will be suppressed when found as duplicate. */
258 if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
259 app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
260 a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
263 // @todo: For now just keep the active alarms in-memory. Use SDL later for persistence
264 a.activeAlarms = append(a.activeAlarms, *newAlarm)
267 func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
268 /* If maximum number of events in alarm history is reached, an error log writing is made,
269 and new alarm indicating the problem is raised. The attempt to add new event time will
272 if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
273 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
274 a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
277 // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
278 a.alarmHistory = append(a.alarmHistory, *newAlarm)
281 func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
282 result, err := json.Marshal(m)
284 app.Logger.Info("json.Marshal failed: %v", err)
288 fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
289 app.Logger.Info("Posting alarm to '%s'", fullUrl)
291 resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
292 if err != nil || resp == nil {
293 app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
299 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
300 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
301 amLabels := models.LabelSet{
302 "status": string(status),
303 "alertname": alarmDef.AlarmText,
304 "severity": string(newAlarm.PerceivedSeverity),
305 "service": fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
306 "system_name": "RIC",
308 amAnnotations := models.LabelSet{
309 "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
310 "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
311 "event_type": alarmDef.EventType,
312 "identifying_info": newAlarm.IdentifyingInfo,
313 "additional_info": newAlarm.AdditionalInfo,
314 "description": fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
315 "instructions": alarmDef.OperationInstructions,
316 "timestamp": fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
319 return amLabels, amAnnotations
322 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
323 cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
324 return client.New(cr, strfmt.Default)
327 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
328 pa := &models.PostableAlert{
330 GeneratorURL: strfmt.URI(""),
333 Annotations: amAnnotations,
335 alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
337 app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
338 ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
340 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
345 func (a *AlarmManager) StatusCB() bool {
347 app.Logger.Info("RMR not ready yet!")
353 func (a *AlarmManager) ConfigChangeCB(configparam string) {
355 a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
356 a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
357 a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
358 a.amHost = viper.GetString("controls.promAlertManager.address")
360 app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
361 app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
362 app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
363 app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
368 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
370 filename := os.Getenv("DEF_FILE")
371 file, err := ioutil.ReadFile(filename)
373 data := RicAlarmDefinitions{}
374 err = json.Unmarshal([]byte(file), &data)
376 for _, alarmDefinition := range data.AlarmDefinitions {
377 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
379 app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
381 app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm %v", alarmDefinition.AlarmId)
382 ricAlarmDefintion := new(alarm.AlarmDefinition)
383 ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
384 ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
385 ricAlarmDefintion.EventType = alarmDefinition.EventType
386 ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
387 ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
388 ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
389 alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
393 app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
396 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
400 func (a *AlarmManager) ReadAlarmInfoFromPersistentVolume() {
401 var alarmpersistentinfo AlarmPersistentInfo
402 byteValue, rerr := ioutil.ReadFile(a.alarmInfoPvFile)
404 app.Logger.Error("ararminfo.json file read error %v", rerr)
406 err := json.Unmarshal(byteValue, &alarmpersistentinfo)
408 app.Logger.Error("alarmpersistentinfo json unmarshal error %v", err)
410 a.uniqueAlarmId = alarmpersistentinfo.UniqueAlarmId
411 a.activeAlarms = make([]AlarmNotification, len(alarmpersistentinfo.ActiveAlarms))
412 a.alarmHistory = make([]AlarmNotification, len(alarmpersistentinfo.AlarmHistory))
413 copy(a.activeAlarms, alarmpersistentinfo.ActiveAlarms)
414 copy(a.alarmHistory, alarmpersistentinfo.AlarmHistory)
419 func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() {
420 var alarmpersistentinfo AlarmPersistentInfo
421 alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
422 alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
423 alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
424 copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
425 copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
426 wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
428 app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
430 werr := ioutil.WriteFile(a.alarmInfoPvFile, wdata, 0777)
432 app.Logger.Error("alarminfo.json file write error %v", werr)
437 func (a *AlarmManager) Run(sdlcheck bool) {
438 app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
439 app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
440 app.Resource.InjectStatusCb(a.StatusCB)
441 app.AddConfigChangeListener(a.ConfigChangeCB)
443 alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
444 a.ReadAlarmDefinitionFromJson()
446 app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
447 app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
448 app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
449 app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
450 app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST")
451 app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET")
452 app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST")
453 app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE")
454 app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET")
455 app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET")
457 // Start background timer for re-raising alerts
458 go a.StartAlertTimer()
459 a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
461 a.ReadAlarmInfoFromPersistentVolume()
463 app.RunWithParams(a, sdlcheck)
466 func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
467 if alertInterval == 0 {
468 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
472 amHost = viper.GetString("controls.promAlertManager.address")
475 return &AlarmManager{
477 postClear: clearAlarm,
479 amBaseUrl: app.Config.GetString("controls.promAlertManager.baseUrl"),
480 amSchemes: []string{app.Config.GetString("controls.promAlertManager.schemes")},
481 alertInterval: alertInterval,
482 activeAlarms: make([]AlarmNotification, 0),
483 alarmHistory: make([]AlarmNotification, 0),
485 maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
486 maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
487 exceededActiveAlarmOn: false,
488 exceededAlarmHistoryOn: false,
489 alarmInfoPvFile: app.Config.GetString("controls.alarmInfoPvFile"),
495 NewAlarmManager("", 0, true).Run(true)