manager/cmd/manager.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "encoding/json"
  25         "fmt"
  26         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  27         app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  28         clientruntime "github.com/go-openapi/runtime/client"
  29         "github.com/go-openapi/strfmt"
  30         "github.com/prometheus/alertmanager/api/v2/client"
  31         "github.com/prometheus/alertmanager/api/v2/client/alert"
  32         "github.com/prometheus/alertmanager/api/v2/models"
  33         "github.com/spf13/viper"
  34         "io/ioutil"
  35         "os"
  36         "time"
  37 )
  38
  39 func (a *AlarmManager) StartAlertTimer() {
  40         tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
  41         for range tick {
  42                 a.mutex.Lock()
  43                 for _, m := range a.activeAlarms {
  44                         app.Logger.Info("Re-raising alarm: %v", m)
  45                         a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
  46                 }
  47                 a.mutex.Unlock()
  48         }
  49 }
  50
  51 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
  52         app.Logger.Info("Message received!")
  53
  54         defer app.Rmr.Free(rp.Mbuf)
  55         switch rp.Mtype {
  56         case alarm.RIC_ALARM_UPDATE:
  57                 a.HandleAlarms(rp)
  58         default:
  59                 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
  60         }
  61
  62         return nil
  63 }
  64
  65 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
  66         var m alarm.AlarmMessage
  67         app.Logger.Info("Received JSON: %s", rp.Payload)
  68         if err := json.Unmarshal(rp.Payload, &m); err != nil {
  69                 app.Logger.Error("json.Unmarshal failed: %v", err)
  70                 return nil, err
  71         }
  72         app.Logger.Info("newAlarm: %v", m)
  73
  74         return a.ProcessAlarm(&m)
  75 }
  76
  77 func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, error) {
  78         a.mutex.Lock()
  79         if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
  80                 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
  81                 a.mutex.Unlock()
  82                 return nil, nil
  83         }
  84
  85         // Suppress duplicate alarms
  86         idx, found := a.IsMatchFound(m.Alarm)
  87         if found && m.AlarmAction == alarm.AlarmActionRaise {
  88                 app.Logger.Info("Duplicate alarm found, suppressing ...")
  89                 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
  90                         // Duplicate with same severity found
  91                         a.mutex.Unlock()
  92                         return nil, nil
  93                 } else {
  94                         // Remove duplicate with different severity
  95                         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
  96                 }
  97         }
  98
  99         // Clear alarm if found from active alarm list
 100         if m.AlarmAction == alarm.AlarmActionClear {
 101                 if found {
 102                         a.alarmHistory = append(a.alarmHistory, *m)
 103                         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 104                         if ((len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false)){
 105                                 app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
 106                                 histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history")
 107                                 histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
 108                                 a.activeAlarms = append(a.activeAlarms, histAlarmMessage)
 109                                 a.alarmHistory = append(a.alarmHistory, histAlarmMessage)
 110                         }
 111                         if ((a.exceededActiveAlarmOn == true) && (m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD)) {
 112                                 a.exceededActiveAlarmOn = false
 113                         }
 114                         if ((a.exceededAlarmHistoryOn == true) && (m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD)) {
 115                                 a.exceededAlarmHistoryOn = false
 116                         }
 117                         if a.postClear {
 118                                 a.mutex.Unlock()
 119                                 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved, m.AlarmTime))
 120                         }
 121                 }
 122                 app.Logger.Info("No matching active alarm found, suppressing ...")
 123                 a.mutex.Unlock()
 124                 return nil, nil
 125         }
 126
 127         // New alarm -> update active alarms and post to Alert Manager
 128         if m.AlarmAction == alarm.AlarmActionRaise {
 129                 a.UpdateAlarmLists(m)
 130                 a.mutex.Unlock()
 131                 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
 132         }
 133
 134         a.mutex.Unlock()
 135         return nil, nil
 136 }
 137
 138 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
 139         for i, m := range a.activeAlarms {
 140                 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
 141                         m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
 142                         return i, true
 143                 }
 144         }
 145         return -1, false
 146 }
 147
 148 func (a *AlarmManager) RemoveAlarm(alarms []alarm.AlarmMessage, i int, listName string) []alarm.AlarmMessage {
 149         app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
 150         copy(alarms[i:], alarms[i+1:])
 151         return alarms[:len(alarms)-1]
 152 }
 153
 154 func (a *AlarmManager) UpdateAlarmLists(newAlarm *alarm.AlarmMessage) {
 155         /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
 156            The attempt to raise the alarm next time will be supressed when found as duplicate. */
 157         if ((len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false)) {
 158                 app.Logger.Error("active alarm count exceeded maxActiveAlarms threshold")
 159                 actAlarm := a.alarmClient.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "active")
 160                 actAlarmMessage := alarm.AlarmMessage{Alarm: actAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
 161                 a.activeAlarms = append(a.activeAlarms, actAlarmMessage)
 162                 a.alarmHistory = append(a.alarmHistory, actAlarmMessage)
 163                 a.exceededActiveAlarmOn = true
 164         }
 165
 166         if ((len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false)) {
 167                 app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
 168                 histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history")
 169                 histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
 170                 a.activeAlarms = append(a.activeAlarms, histAlarmMessage)
 171                 a.alarmHistory = append(a.alarmHistory, histAlarmMessage)
 172                 a.exceededAlarmHistoryOn = true
 173         }
 174
 175         // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
 176         a.activeAlarms = append(a.activeAlarms, *newAlarm)
 177         a.alarmHistory = append(a.alarmHistory, *newAlarm)
 178 }
 179
 180 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
 181         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 182         amLabels := models.LabelSet{
 183                 "status":      string(status),
 184                 "alertname":   alarmDef.AlarmText,
 185                 "severity":    string(newAlarm.PerceivedSeverity),
 186                 "service":     fmt.Sprintf("%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
 187                 "system_name": fmt.Sprintf("RIC:%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
 188         }
 189         amAnnotations := models.LabelSet{
 190                 "alarm_id":        fmt.Sprintf("%d", alarmDef.AlarmId),
 191                 "description":     fmt.Sprintf("%d:%s:%s", newAlarm.SpecificProblem, newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
 192                 "additional_info": newAlarm.AdditionalInfo,
 193                 "summary":         alarmDef.EventType,
 194                 "instructions":    alarmDef.OperationInstructions,
 195                 "timestamp":       fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
 196         }
 197
 198         return amLabels, amAnnotations
 199 }
 200
 201 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
 202         cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
 203         return client.New(cr, strfmt.Default)
 204 }
 205
 206 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
 207         pa := &models.PostableAlert{
 208                 Alert: models.Alert{
 209                         GeneratorURL: strfmt.URI(""),
 210                         Labels:       amLabels,
 211                 },
 212                 Annotations: amAnnotations,
 213         }
 214         alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
 215
 216         app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
 217         ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
 218         if err != nil {
 219                 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
 220         }
 221         return ok, err
 222 }
 223
 224 func (a *AlarmManager) StatusCB() bool {
 225         if !a.rmrReady {
 226                 app.Logger.Info("RMR not ready yet!")
 227         }
 228
 229         return a.rmrReady
 230 }
 231
 232 func (a *AlarmManager) ConfigChangeCB(configparam string) {
 233
 234         a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
 235         a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
 236         a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 237         a.amHost = viper.GetString("controls.promAlertManager.address")
 238
 239         app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
 240         app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
 241         app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
 242         app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
 243
 244         return
 245 }
 246
 247 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
 248
 249         filename := os.Getenv("DEF_FILE")
 250         file, err := ioutil.ReadFile(filename)
 251         if err == nil {
 252                 data := RicAlarmDefinitions{}
 253                 err = json.Unmarshal([]byte(file), &data)
 254                 if err == nil {
 255                         for _, alarmDefinition := range data.AlarmDefinitions {
 256                                 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
 257                                 if exists {
 258                                         app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
 259                                 } else {
 260                                         app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm  %v", alarmDefinition.AlarmId)
 261                                         ricAlarmDefintion := new(alarm.AlarmDefinition)
 262                                         ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
 263                                         ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
 264                                         ricAlarmDefintion.EventType = alarmDefinition.EventType
 265                                         ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
 266                                         alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
 267                                 }
 268                         }
 269                 } else {
 270                         app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
 271                 }
 272         } else {
 273                 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
 274         }
 275 }
 276
 277 func (a *AlarmManager) Run(sdlcheck bool) {
 278         app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
 279         app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
 280         app.Resource.InjectStatusCb(a.StatusCB)
 281         app.AddConfigChangeListener(a.ConfigChangeCB)
 282
 283         alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
 284         a.ReadAlarmDefinitionFromJson()
 285
 286         app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
 287         app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
 288         app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
 289         app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
 290         app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST")
 291         app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET")
 292         app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST")
 293         app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE")
 294         app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET")
 295         app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET")
 296
 297         // Start background timer for re-raising alerts
 298         a.postClear = sdlcheck
 299         go a.StartAlertTimer()
 300         a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
 301
 302         app.RunWithParams(a, sdlcheck)
 303 }
 304
 305 func NewAlarmManager(amHost string, alertInterval int) *AlarmManager {
 306         if alertInterval == 0 {
 307                 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 308         }
 309
 310         if amHost == "" {
 311                 amHost = viper.GetString("controls.promAlertManager.address")
 312         }
 313
 314         return &AlarmManager{
 315                 rmrReady:        false,
 316                 amHost:          amHost,
 317                 amBaseUrl:       viper.GetString("controls.promAlertManager.baseUrl"),
 318                 amSchemes:       []string{viper.GetString("controls.promAlertManager.schemes")},
 319                 alertInterval:   alertInterval,
 320                 activeAlarms:    make([]alarm.AlarmMessage, 0),
 321                 alarmHistory:    make([]alarm.AlarmMessage, 0),
 322                 maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
 323                 maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
 324                 exceededActiveAlarmOn:  false,
 325                 exceededAlarmHistoryOn: false,
 326         }
 327 }
 328
 329 // Main function
 330 func main() {
 331         NewAlarmManager("", 0).Run(true)
 332 }