manager/cmd/manager.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io/ioutil"
  28         "net/http"
  29         "os"
  30         "time"
  31
  32         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  33         app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  34         clientruntime "github.com/go-openapi/runtime/client"
  35         "github.com/go-openapi/strfmt"
  36         "github.com/prometheus/alertmanager/api/v2/client"
  37         "github.com/prometheus/alertmanager/api/v2/client/alert"
  38         "github.com/prometheus/alertmanager/api/v2/models"
  39         "github.com/spf13/viper"
  40 )
  41
  42 func (a *AlarmManager) ClearExpiredAlarms(m AlarmNotification, idx int, mLocked bool) bool {
  43         d, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]
  44         if !ok || d.TimeToLive == 0 {
  45                 return false
  46         }
  47
  48         elapsed := (time.Now().UnixNano() - m.AlarmTime) / 1e9
  49         if int(elapsed) >= d.TimeToLive {
  50                 app.Logger.Info("Alarm (sp=%d id=%d) with TTL=%d expired, clearing ...", m.Alarm.SpecificProblem, m.AlarmId, d.TimeToLive)
  51
  52                 m.AlarmAction = alarm.AlarmActionClear
  53                 m.AlarmTime = time.Now().UnixNano()
  54
  55                 if !mLocked { // For testing purpose
  56                         a.mutex.Lock()
  57                 }
  58                 a.ProcessClearAlarm(&m, d, idx)
  59                 return true
  60         }
  61         return false
  62 }
  63
  64 func (a *AlarmManager) StartTTLTimer(interval int) {
  65         tick := time.Tick(time.Duration(interval) * time.Second)
  66         for range tick {
  67                 a.mutex.Lock()
  68                 for idx, m := range a.activeAlarms {
  69                         if a.ClearExpiredAlarms(m, idx, true) {
  70                                 a.mutex.Lock() // ClearExpiredAlarms unlocks the mutex, so re-lock here
  71                                 continue
  72                         }
  73                 }
  74                 a.mutex.Unlock()
  75         }
  76 }
  77
  78 func (a *AlarmManager) StartAlertTimer() {
  79         tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
  80         for range tick {
  81                 a.mutex.Lock()
  82                 for _, m := range a.activeAlarms {
  83                         app.Logger.Info("Re-raising alarm: %v", m)
  84                         a.PostAlert(a.GenerateAlertLabels(m.AlarmId, m.Alarm, AlertStatusActive, m.AlarmTime))
  85                 }
  86                 a.mutex.Unlock()
  87         }
  88 }
  89
  90 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
  91         app.Logger.Info("Message received!")
  92
  93         defer app.Rmr.Free(rp.Mbuf)
  94         switch rp.Mtype {
  95         case alarm.RIC_ALARM_UPDATE:
  96                 a.HandleAlarms(rp)
  97         default:
  98                 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
  99         }
 100
 101         return nil
 102 }
 103
 104 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
 105         var m alarm.AlarmMessage
 106         app.Logger.Info("Received JSON: %s", rp.Payload)
 107         if err := json.Unmarshal(rp.Payload, &m); err != nil {
 108                 app.Logger.Error("json.Unmarshal failed: %v", err)
 109                 return nil, err
 110         }
 111         app.Logger.Info("newAlarm: %v", m)
 112
 113         return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
 114 }
 115
 116 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 117         a.mutex.Lock()
 118         alarmDef := &alarm.AlarmDefinition{}
 119         var ok bool
 120         if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
 121                 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
 122                 a.mutex.Unlock()
 123                 return nil, nil
 124         }
 125
 126         idx, found := a.IsMatchFound(m.Alarm)
 127         // Suppress duplicate alarms
 128         if found && m.AlarmAction == alarm.AlarmActionRaise {
 129                 app.Logger.Info("Duplicate alarm found, suppressing ...")
 130                 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
 131                         // Duplicate with same severity found
 132                         a.mutex.Unlock()
 133                         return nil, nil
 134                 } else {
 135                         // Remove duplicate with different severity
 136                         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 137                 }
 138         }
 139
 140         // Clear alarm if found from active alarm list
 141         if found && m.AlarmAction == alarm.AlarmActionClear {
 142                 return a.ProcessClearAlarm(m, alarmDef, idx)
 143         }
 144
 145         // New alarm -> update active alarms and post to Alert Manager
 146         if m.AlarmAction == alarm.AlarmActionRaise {
 147                 return a.ProcessRaiseAlarm(m, alarmDef)
 148         }
 149
 150         a.mutex.Unlock()
 151         return nil, nil
 152 }
 153
 154 func (a *AlarmManager) ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) (*alert.PostAlertsOK, error) {
 155         app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 156
 157         // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
 158         m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
 159         a.UpdateAlarmFields(a.GenerateAlarmId(), m)
 160         a.UpdateActiveAlarmList(m)
 161         a.mutex.Unlock()
 162
 163         if alarmDef.RaiseDelay > 0 {
 164                 timerDelay(alarmDef.RaiseDelay)
 165                 a.mutex.Lock()
 166                 // Alarm may have been deleted from active alarms table during delay or table index may have changed
 167                 idx, found := a.IsMatchFound(m.Alarm)
 168                 if found {
 169                         // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
 170                         a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
 171                         app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 172                         a.mutex.Unlock()
 173                 } else {
 174                         app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
 175                         a.mutex.Unlock()
 176                         return nil, nil
 177                 }
 178         }
 179
 180         m.AlarmDefinition.RaiseDelay = 0
 181         a.UpdateAlarmHistoryList(m)
 182         a.WriteAlarmInfoToPersistentVolume()
 183
 184         // Send alarm notification to NOMA, if enabled
 185         if app.Config.GetBool("controls.noma.enabled") {
 186                 return a.PostAlarm(m)
 187         }
 188         return a.PostAlert(a.GenerateAlertLabels(m.AlarmId, m.Alarm, AlertStatusActive, m.AlarmTime))
 189 }
 190
 191 func (a *AlarmManager) ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) (*alert.PostAlertsOK, error) {
 192         app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 193         if alarmDef.ClearDelay > 0 {
 194                 a.mutex.Unlock()
 195                 timerDelay(alarmDef.ClearDelay)
 196                 app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 197                 a.mutex.Lock()
 198                 // Another alarm clear may have happened during delay and active alarms table index changed
 199                 var found bool
 200                 idx, found = a.IsMatchFound(m.Alarm)
 201                 if !found {
 202                         a.mutex.Unlock()
 203                         return nil, nil
 204                 }
 205         }
 206         a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
 207         a.alarmHistory = append(a.alarmHistory, *m)
 208         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 209         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 210                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 211                 a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 212         }
 213
 214         if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
 215                 a.exceededActiveAlarmOn = false
 216         }
 217
 218         if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
 219                 a.exceededAlarmHistoryOn = false
 220         }
 221         a.WriteAlarmInfoToPersistentVolume()
 222
 223         a.mutex.Unlock()
 224         if a.postClear && app.Config.GetBool("controls.noma.enabled") {
 225                 m.PerceivedSeverity = alarm.SeverityCleared
 226                 return a.PostAlarm(m)
 227         }
 228         return nil, nil
 229 }
 230
 231 func timerDelay(delay int) {
 232         timer := time.NewTimer(time.Duration(delay) * time.Second)
 233         <-timer.C
 234 }
 235
 236 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
 237         for i, m := range a.activeAlarms {
 238                 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
 239                         m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
 240                         return i, true
 241                 }
 242         }
 243         return -1, false
 244 }
 245
 246 func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
 247         app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
 248         copy(alarms[i:], alarms[i+1:])
 249         return alarms[:len(alarms)-1]
 250 }
 251
 252 func (a *AlarmManager) GenerateAlarmId() int {
 253         a.uniqueAlarmId++ // @todo: generate a unique ID
 254         return a.uniqueAlarmId
 255 }
 256
 257 func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
 258         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 259         newAlarm.AlarmId = alarmId
 260         newAlarm.AlarmText = alarmDef.AlarmText
 261         newAlarm.EventType = alarmDef.EventType
 262 }
 263
 264 func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
 265         thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
 266         thresholdMessage := alarm.AlarmMessage{
 267                 Alarm:       thresholdAlarm,
 268                 AlarmAction: alarm.AlarmActionRaise,
 269                 AlarmTime:   time.Now().UnixNano(),
 270         }
 271         alarmDef := alarm.RICAlarmDefinitions[sp]
 272         alarmId := a.GenerateAlarmId()
 273         alarmDef.AlarmId = alarmId
 274         a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, *alarmDef})
 275         a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, *alarmDef})
 276
 277         return true
 278 }
 279
 280 func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
 281         /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
 282            The attempt to raise the alarm next time will be suppressed when found as duplicate. */
 283         if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
 284                 app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
 285                 a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
 286         }
 287
 288         // @todo: For now just keep the  active alarms in-memory. Use SDL later for persistence
 289         a.activeAlarms = append(a.activeAlarms, *newAlarm)
 290 }
 291
 292 func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
 293         /* If maximum number of events in alarm history is reached, an error log writing is made,
 294            and new alarm indicating the problem is raised. The attempt to add new event time will
 295            be suppressed */
 296
 297         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 298                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 299                 a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 300         }
 301
 302         // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
 303         a.alarmHistory = append(a.alarmHistory, *newAlarm)
 304 }
 305
 306 func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 307         result, err := json.Marshal(m)
 308         if err != nil {
 309                 app.Logger.Info("json.Marshal failed: %v", err)
 310                 return nil, err
 311         }
 312
 313         fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
 314         app.Logger.Info("Posting alarm to '%s'", fullUrl)
 315
 316         resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
 317         if err != nil || resp == nil {
 318                 app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
 319         }
 320
 321         return nil, err
 322 }
 323
 324 func (a *AlarmManager) GenerateAlertLabels(alarmId int, newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
 325         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 326         amLabels := models.LabelSet{
 327                 "status":      string(status),
 328                 "alertname":   alarmDef.AlarmText,
 329                 "severity":    string(newAlarm.PerceivedSeverity),
 330                 "service":     fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
 331                 "system_name": "RIC",
 332         }
 333         amAnnotations := models.LabelSet{
 334                 "alarm_id":         fmt.Sprintf("%d", alarmId),
 335                 "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
 336                 "event_type":       alarmDef.EventType,
 337                 "identifying_info": newAlarm.IdentifyingInfo,
 338                 "additional_info":  newAlarm.AdditionalInfo,
 339                 "description":      fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
 340                 "instructions":     alarmDef.OperationInstructions,
 341                 "timestamp":        fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
 342         }
 343
 344         return amLabels, amAnnotations
 345 }
 346
 347 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
 348         cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
 349         return client.New(cr, strfmt.Default)
 350 }
 351
 352 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
 353         pa := &models.PostableAlert{
 354                 Alert: models.Alert{
 355                         GeneratorURL: strfmt.URI(""),
 356                         Labels:       amLabels,
 357                 },
 358                 Annotations: amAnnotations,
 359         }
 360         alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
 361
 362         app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
 363         ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
 364         if err != nil {
 365                 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
 366         }
 367         return ok, err
 368 }
 369
 370 func (a *AlarmManager) StatusCB() bool {
 371         if !a.rmrReady {
 372                 app.Logger.Info("RMR not ready yet!")
 373         }
 374         return a.rmrReady
 375 }
 376
 377 func (a *AlarmManager) ConfigChangeCB(configparam string) {
 378         a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
 379         if a.maxActiveAlarms == 0 {
 380                 a.maxActiveAlarms = 5000
 381         }
 382
 383         a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
 384         if a.maxAlarmHistory == 0 {
 385                 a.maxAlarmHistory = 20000
 386         }
 387
 388         a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 389         a.amHost = viper.GetString("controls.promAlertManager.address")
 390
 391         app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
 392         app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
 393         app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
 394         app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
 395
 396         return
 397 }
 398
 399 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
 400
 401         filename := os.Getenv("DEF_FILE")
 402         file, err := ioutil.ReadFile(filename)
 403         if err == nil {
 404                 data := RicAlarmDefinitions{}
 405                 err = json.Unmarshal([]byte(file), &data)
 406                 if err == nil {
 407                         for _, alarmDefinition := range data.AlarmDefinitions {
 408                                 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
 409                                 if exists {
 410                                         app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
 411                                 } else {
 412                                         app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm  %v", alarmDefinition.AlarmId)
 413                                         ricAlarmDefintion := new(alarm.AlarmDefinition)
 414                                         ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
 415                                         ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
 416                                         ricAlarmDefintion.EventType = alarmDefinition.EventType
 417                                         ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
 418                                         ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
 419                                         ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
 420                                         ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive
 421                                         alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
 422                                 }
 423                         }
 424                 } else {
 425                         app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
 426                 }
 427         } else {
 428                 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
 429         }
 430 }
 431
 432 func (a *AlarmManager) ReadAlarmInfoFromPersistentVolume() {
 433         var alarmpersistentinfo AlarmPersistentInfo
 434         byteValue, rerr := ioutil.ReadFile(a.alarmInfoPvFile)
 435         if rerr != nil {
 436                 app.Logger.Error("ararminfo.json file read error %v", rerr)
 437         } else {
 438                 err := json.Unmarshal(byteValue, &alarmpersistentinfo)
 439                 if err != nil {
 440                         app.Logger.Error("alarmpersistentinfo json unmarshal error %v", err)
 441                 } else {
 442                         a.uniqueAlarmId = alarmpersistentinfo.UniqueAlarmId
 443                         a.activeAlarms = make([]AlarmNotification, len(alarmpersistentinfo.ActiveAlarms))
 444                         a.alarmHistory = make([]AlarmNotification, len(alarmpersistentinfo.AlarmHistory))
 445                         copy(a.activeAlarms, alarmpersistentinfo.ActiveAlarms)
 446                         copy(a.alarmHistory, alarmpersistentinfo.AlarmHistory)
 447                 }
 448         }
 449 }
 450
 451 func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() {
 452         var alarmpersistentinfo AlarmPersistentInfo
 453         alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
 454         alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
 455         alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
 456
 457         copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
 458         copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
 459
 460         wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
 461         if err != nil {
 462                 app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
 463         } else {
 464                 werr := ioutil.WriteFile(a.alarmInfoPvFile, wdata, 0777)
 465                 if werr != nil {
 466                         app.Logger.Error("alarminfo.json file write error %v", werr)
 467                 }
 468         }
 469 }
 470
 471 func (a *AlarmManager) Run(sdlcheck bool, ttlInterval int) {
 472         app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
 473         app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
 474         app.Resource.InjectStatusCb(a.StatusCB)
 475         app.AddConfigChangeListener(a.ConfigChangeCB)
 476
 477         alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
 478         a.ReadAlarmDefinitionFromJson()
 479
 480         a.InjectRoutes()
 481
 482         // Start background timer for re-raising alerts
 483         go a.StartAlertTimer()
 484         go a.StartTTLTimer(ttlInterval)
 485
 486         a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
 487
 488         a.ReadAlarmInfoFromPersistentVolume()
 489
 490         app.RunWithParams(a, sdlcheck)
 491 }
 492
 493 func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
 494         if alertInterval == 0 {
 495                 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 496         }
 497
 498         if amHost == "" {
 499                 amHost = viper.GetString("controls.promAlertManager.address")
 500         }
 501
 502         maxActiveAlarms := app.Config.GetInt("controls.maxActiveAlarms")
 503         if maxActiveAlarms == 0 {
 504                 maxActiveAlarms = 5000
 505         }
 506
 507         maxAlarmHistory := app.Config.GetInt("controls.maxAlarmHistory")
 508         if maxAlarmHistory == 0 {
 509                 maxAlarmHistory = 20000
 510         }
 511
 512         return &AlarmManager{
 513                 rmrReady:               false,
 514                 postClear:              clearAlarm,
 515                 amHost:                 amHost,
 516                 amBaseUrl:              app.Config.GetString("controls.promAlertManager.baseUrl"),
 517                 amSchemes:              []string{app.Config.GetString("controls.promAlertManager.schemes")},
 518                 alertInterval:          alertInterval,
 519                 activeAlarms:           make([]AlarmNotification, 0),
 520                 alarmHistory:           make([]AlarmNotification, 0),
 521                 uniqueAlarmId:          0,
 522                 maxActiveAlarms:        maxActiveAlarms,
 523                 maxAlarmHistory:        maxAlarmHistory,
 524                 exceededActiveAlarmOn:  false,
 525                 exceededAlarmHistoryOn: false,
 526                 alarmInfoPvFile:        app.Config.GetString("controls.alarmInfoPvFile"),
 527         }
 528 }
 529
 530 // Main function
 531 func main() {
 532         NewAlarmManager("", 0, true).Run(true, 10)
 533 }