2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
32 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
33 app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
34 clientruntime "github.com/go-openapi/runtime/client"
35 "github.com/go-openapi/strfmt"
36 "github.com/prometheus/alertmanager/api/v2/client"
37 "github.com/prometheus/alertmanager/api/v2/client/alert"
38 "github.com/prometheus/alertmanager/api/v2/models"
39 "github.com/spf13/viper"
42 func (a *AlarmManager) StartAlertTimer() {
43 tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
46 for _, m := range a.activeAlarms {
47 app.Logger.Info("Re-raising alarm: %v", m)
48 a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
54 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
55 app.Logger.Info("Message received!")
57 defer app.Rmr.Free(rp.Mbuf)
59 case alarm.RIC_ALARM_UPDATE:
62 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
68 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
69 var m alarm.AlarmMessage
70 app.Logger.Info("Received JSON: %s", rp.Payload)
71 if err := json.Unmarshal(rp.Payload, &m); err != nil {
72 app.Logger.Error("json.Unmarshal failed: %v", err)
75 app.Logger.Info("newAlarm: %v", m)
77 return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
80 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
82 alarmDef := &alarm.AlarmDefinition{}
84 if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
85 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
90 idx, found := a.IsMatchFound(m.Alarm)
91 // Suppress duplicate alarms
92 if found && m.AlarmAction == alarm.AlarmActionRaise {
93 app.Logger.Info("Duplicate alarm found, suppressing ...")
94 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
95 // Duplicate with same severity found
99 // Remove duplicate with different severity
100 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
104 // Clear alarm if found from active alarm list
105 if m.AlarmAction == alarm.AlarmActionClear {
107 app.Logger.Info("No matching active alarm found, suppressing ...")
112 if a.ProcessClearAlarm(m, alarmDef, idx) == false {
118 app.Logger.Info("Sending clear notification disabled!")
122 // Send alarm notification to NOMA, if enabled
123 if app.Config.GetBool("controls.noma.enabled") {
124 m.PerceivedSeverity = alarm.SeverityCleared
125 return a.PostAlarm(m)
129 // New alarm -> update active alarms and post to Alert Manager
130 if m.AlarmAction == alarm.AlarmActionRaise {
131 if a.ProcessRaiseAlarm(m, alarmDef) == false {
134 // Send alarm notification to NOMA, if enabled
135 if app.Config.GetBool("controls.noma.enabled") {
136 return a.PostAlarm(m)
138 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
145 func (a *AlarmManager)ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) bool {
146 app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
147 // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
148 m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
149 a.UpdateAlarmFields(a.GenerateAlarmId(), m)
150 a.UpdateActiveAlarmList(m)
152 if alarmDef.RaiseDelay > 0 {
153 timerDelay(alarmDef.RaiseDelay)
155 // Alarm may have been deleted from active alarms table during delay or table index may have changed
156 idx, found := a.IsMatchFound(m.Alarm)
158 // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
159 a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
160 app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
163 app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
168 m.AlarmDefinition.RaiseDelay = 0
169 a.UpdateAlarmHistoryList(m)
170 a.WriteAlarmInfoToPersistentVolume()
174 func (a *AlarmManager)ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) bool {
175 app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
176 if alarmDef.ClearDelay > 0 {
178 timerDelay(alarmDef.ClearDelay)
179 app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
181 // Another alarm clear may have happened during delay and active alarms table index changed
183 idx, found = a.IsMatchFound(m.Alarm)
185 app.Logger.Debug("Alarm not anymore in the active alarms table. AlarmNotification = %v", *m)
190 a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
191 a.alarmHistory = append(a.alarmHistory, *m)
192 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
193 if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
194 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
195 a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
198 if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
199 a.exceededActiveAlarmOn = false
202 if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
203 a.exceededAlarmHistoryOn = false
205 a.WriteAlarmInfoToPersistentVolume()
209 func timerDelay(delay int) {
210 timer := time.NewTimer(time.Duration(delay) * time.Second)
214 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
215 for i, m := range a.activeAlarms {
216 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
217 m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
224 func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
225 app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
226 copy(alarms[i:], alarms[i+1:])
227 return alarms[:len(alarms)-1]
230 func (a *AlarmManager) GenerateAlarmId() int {
231 a.uniqueAlarmId++ // @todo: generate a unique ID
232 return a.uniqueAlarmId
235 func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
236 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
237 newAlarm.AlarmId = alarmId
238 newAlarm.AlarmText = alarmDef.AlarmText
239 newAlarm.EventType = alarmDef.EventType
242 func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
243 thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
244 thresholdMessage := alarm.AlarmMessage{
245 Alarm: thresholdAlarm,
246 AlarmAction: alarm.AlarmActionRaise,
247 AlarmTime: (time.Now().UnixNano()),
249 alarmDef := alarm.RICAlarmDefinitions[sp]
250 alarmId := a.GenerateAlarmId()
251 alarmDef.AlarmId = alarmId
252 a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, *alarmDef})
253 a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, *alarmDef})
258 func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
259 /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
260 The attempt to raise the alarm next time will be suppressed when found as duplicate. */
261 if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
262 app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
263 a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
266 // @todo: For now just keep the active alarms in-memory. Use SDL later for persistence
267 a.activeAlarms = append(a.activeAlarms, *newAlarm)
270 func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
271 /* If maximum number of events in alarm history is reached, an error log writing is made,
272 and new alarm indicating the problem is raised. The attempt to add new event time will
275 if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
276 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
277 a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
280 // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
281 a.alarmHistory = append(a.alarmHistory, *newAlarm)
284 func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
285 result, err := json.Marshal(m)
287 app.Logger.Info("json.Marshal failed: %v", err)
291 fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
292 app.Logger.Info("Posting alarm to '%s'", fullUrl)
294 resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
295 if err != nil || resp == nil {
296 app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
302 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
303 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
304 amLabels := models.LabelSet{
305 "status": string(status),
306 "alertname": alarmDef.AlarmText,
307 "severity": string(newAlarm.PerceivedSeverity),
308 "service": fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
309 "system_name": "RIC",
311 amAnnotations := models.LabelSet{
312 "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
313 "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
314 "event_type": alarmDef.EventType,
315 "identifying_info": newAlarm.IdentifyingInfo,
316 "additional_info": newAlarm.AdditionalInfo,
317 "description": fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
318 "instructions": alarmDef.OperationInstructions,
319 "timestamp": fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
322 return amLabels, amAnnotations
325 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
326 cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
327 return client.New(cr, strfmt.Default)
330 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
331 pa := &models.PostableAlert{
333 GeneratorURL: strfmt.URI(""),
336 Annotations: amAnnotations,
338 alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
340 app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
341 ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
343 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
348 func (a *AlarmManager) StatusCB() bool {
350 app.Logger.Info("RMR not ready yet!")
356 func (a *AlarmManager) ConfigChangeCB(configparam string) {
358 a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
359 a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
360 a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
361 a.amHost = viper.GetString("controls.promAlertManager.address")
363 app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
364 app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
365 app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
366 app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
371 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
373 filename := os.Getenv("DEF_FILE")
374 file, err := ioutil.ReadFile(filename)
376 data := RicAlarmDefinitions{}
377 err = json.Unmarshal([]byte(file), &data)
379 for _, alarmDefinition := range data.AlarmDefinitions {
380 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
382 app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
384 app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm %v", alarmDefinition.AlarmId)
385 ricAlarmDefintion := new(alarm.AlarmDefinition)
386 ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
387 ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
388 ricAlarmDefintion.EventType = alarmDefinition.EventType
389 ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
390 ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
391 ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
392 alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
396 app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
399 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
403 func (a *AlarmManager) ReadAlarmInfoFromPersistentVolume() {
404 var alarmpersistentinfo AlarmPersistentInfo
405 byteValue, rerr := ioutil.ReadFile(a.alarmInfoPvFile)
407 app.Logger.Error("ararminfo.json file read error %v", rerr)
409 err := json.Unmarshal(byteValue, &alarmpersistentinfo)
411 app.Logger.Error("alarmpersistentinfo json unmarshal error %v", err)
413 a.uniqueAlarmId = alarmpersistentinfo.UniqueAlarmId
414 a.activeAlarms = make([]AlarmNotification, len(alarmpersistentinfo.ActiveAlarms))
415 a.alarmHistory = make([]AlarmNotification, len(alarmpersistentinfo.AlarmHistory))
416 copy(a.activeAlarms, alarmpersistentinfo.ActiveAlarms)
417 copy(a.alarmHistory, alarmpersistentinfo.AlarmHistory)
422 func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() {
423 var alarmpersistentinfo AlarmPersistentInfo
424 alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
425 alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
426 alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
427 copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
428 copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
429 wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
431 app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
433 werr := ioutil.WriteFile(a.alarmInfoPvFile, wdata, 0777)
435 app.Logger.Error("alarminfo.json file write error %v", werr)
440 func (a *AlarmManager) Run(sdlcheck bool) {
441 app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
442 app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
443 app.Resource.InjectStatusCb(a.StatusCB)
444 app.AddConfigChangeListener(a.ConfigChangeCB)
446 alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
447 a.ReadAlarmDefinitionFromJson()
449 app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
450 app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
451 app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
452 app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
453 app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST")
454 app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET")
455 app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST")
456 app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE")
457 app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET")
458 app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET")
460 // Start background timer for re-raising alerts
461 go a.StartAlertTimer()
462 a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
464 a.ReadAlarmInfoFromPersistentVolume()
466 app.RunWithParams(a, sdlcheck)
469 func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
470 if alertInterval == 0 {
471 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
475 amHost = viper.GetString("controls.promAlertManager.address")
478 return &AlarmManager{
480 postClear: clearAlarm,
482 amBaseUrl: app.Config.GetString("controls.promAlertManager.baseUrl"),
483 amSchemes: []string{app.Config.GetString("controls.promAlertManager.schemes")},
484 alertInterval: alertInterval,
485 activeAlarms: make([]AlarmNotification, 0),
486 alarmHistory: make([]AlarmNotification, 0),
488 maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
489 maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
490 exceededActiveAlarmOn: false,
491 exceededAlarmHistoryOn: false,
492 alarmInfoPvFile: app.Config.GetString("controls.alarmInfoPvFile"),
498 NewAlarmManager("", 0, true).Run(true)