manager/cmd/manager.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io/ioutil"
  28         "net/http"
  29         "os"
  30         "time"
  31
  32         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  33         app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  34         clientruntime "github.com/go-openapi/runtime/client"
  35         "github.com/go-openapi/strfmt"
  36         "github.com/prometheus/alertmanager/api/v2/client"
  37         "github.com/prometheus/alertmanager/api/v2/client/alert"
  38         "github.com/prometheus/alertmanager/api/v2/models"
  39         "github.com/spf13/viper"
  40 )
  41
  42 func (a *AlarmManager) ClearExpiredAlarms(m AlarmNotification, idx int, mLocked bool) bool {
  43         d, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]
  44         if !ok || d.TimeToLive == 0 {
  45                 return false
  46         }
  47
  48         elapsed := (time.Now().UnixNano() - m.AlarmTime) / 1e9
  49         if int(elapsed) >= d.TimeToLive {
  50                 app.Logger.Info("Alarm (sp=%d id=%d) with TTL=%d expired, clearing ...", m.Alarm.SpecificProblem, m.AlarmId, d.TimeToLive)
  51
  52                 m.AlarmAction = alarm.AlarmActionClear
  53                 m.AlarmTime = time.Now().UnixNano()
  54
  55                 if !mLocked { // For testing purpose
  56                         a.mutex.Lock()
  57                 }
  58                 a.ProcessClearAlarm(&m, d, idx)
  59                 return true
  60         }
  61         return false
  62 }
  63
  64 func (a *AlarmManager) StartTTLTimer(interval int) {
  65         tick := time.Tick(time.Duration(interval) * time.Second)
  66         for range tick {
  67                 a.mutex.Lock()
  68                 for idx, m := range a.activeAlarms {
  69                         if a.ClearExpiredAlarms(m, idx, true) {
  70                                 a.mutex.Lock() // ClearExpiredAlarms unlocks the mutex, so re-lock here
  71                                 continue
  72                         }
  73                 }
  74                 a.mutex.Unlock()
  75         }
  76 }
  77
  78 func (a *AlarmManager) StartAlertTimer() {
  79         tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
  80         for range tick {
  81                 a.mutex.Lock()
  82                 for _, m := range a.activeAlarms {
  83                         app.Logger.Info("Re-raising alarm: %v", m)
  84                         a.PostAlert(a.GenerateAlertLabels(m.AlarmId, m.Alarm, AlertStatusActive, m.AlarmTime))
  85                 }
  86                 a.mutex.Unlock()
  87         }
  88 }
  89
  90 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
  91         app.Logger.Info("Message received!")
  92
  93         defer app.Rmr.Free(rp.Mbuf)
  94         switch rp.Mtype {
  95         case alarm.RIC_ALARM_UPDATE:
  96                 a.HandleAlarms(rp)
  97         default:
  98                 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
  99         }
 100
 101         return nil
 102 }
 103
 104 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
 105         var m alarm.AlarmMessage
 106         app.Logger.Info("Received JSON: %s", rp.Payload)
 107         if err := json.Unmarshal(rp.Payload, &m); err != nil {
 108                 app.Logger.Error("json.Unmarshal failed: %v", err)
 109                 return nil, err
 110         }
 111         app.Logger.Info("newAlarm: %v", m)
 112
 113         return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
 114 }
 115
 116 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 117         a.mutex.Lock()
 118         alarmDef := &alarm.AlarmDefinition{}
 119         var ok bool
 120         if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
 121                 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
 122                 a.mutex.Unlock()
 123                 return nil, nil
 124         }
 125
 126         idx, found := a.IsMatchFound(m.Alarm)
 127         // Suppress duplicate alarms
 128         if found && m.AlarmAction == alarm.AlarmActionRaise {
 129                 app.Logger.Info("Duplicate alarm found, suppressing ...")
 130                 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
 131                         // Duplicate with same severity found
 132                         a.mutex.Unlock()
 133                         return nil, nil
 134                 } else {
 135                         // Remove duplicate with different severity
 136                         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 137                 }
 138         }
 139
 140         // Clear alarm if found from active alarm list
 141         if found && m.AlarmAction == alarm.AlarmActionClear {
 142                 return a.ProcessClearAlarm(m, alarmDef, idx)
 143         }
 144
 145         // New alarm -> update active alarms and post to Alert Manager
 146         if m.AlarmAction == alarm.AlarmActionRaise {
 147                 return a.ProcessRaiseAlarm(m, alarmDef)
 148         }
 149
 150         a.mutex.Unlock()
 151         return nil, nil
 152 }
 153
 154 func (a *AlarmManager) ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) (*alert.PostAlertsOK, error) {
 155         app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 156
 157         // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
 158         m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
 159         a.UpdateAlarmFields(a.GenerateAlarmId(), m)
 160         a.UpdateActiveAlarmList(m)
 161         a.mutex.Unlock()
 162
 163         if alarmDef.RaiseDelay > 0 {
 164                 timerDelay(alarmDef.RaiseDelay)
 165                 a.mutex.Lock()
 166                 // Alarm may have been deleted from active alarms table during delay or table index may have changed
 167                 idx, found := a.IsMatchFound(m.Alarm)
 168                 if found {
 169                         // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
 170                         a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
 171                         app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 172                         a.mutex.Unlock()
 173                 } else {
 174                         app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
 175                         a.mutex.Unlock()
 176                         return nil, nil
 177                 }
 178         }
 179
 180         m.AlarmDefinition.RaiseDelay = 0
 181         a.UpdateAlarmHistoryList(m)
 182         a.WriteAlarmInfoToPersistentVolume()
 183
 184         // Send alarm notification to NOMA, if enabled
 185         if app.Config.GetBool("controls.noma.enabled") {
 186                 return a.PostAlarm(m)
 187         }
 188         return a.PostAlert(a.GenerateAlertLabels(m.AlarmId, m.Alarm, AlertStatusActive, m.AlarmTime))
 189 }
 190
 191 func (a *AlarmManager) ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) (*alert.PostAlertsOK, error) {
 192         app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 193         if alarmDef.ClearDelay > 0 {
 194                 a.mutex.Unlock()
 195                 timerDelay(alarmDef.ClearDelay)
 196                 app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 197                 a.mutex.Lock()
 198                 // Another alarm clear may have happened during delay and active alarms table index changed
 199                 var found bool
 200                 idx, found = a.IsMatchFound(m.Alarm)
 201                 if !found {
 202                         a.mutex.Unlock()
 203                         return nil, nil
 204                 }
 205         }
 206         a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
 207         a.alarmHistory = append(a.alarmHistory, *m)
 208         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 209         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 210                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 211                 a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 212         }
 213
 214         if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
 215                 a.exceededActiveAlarmOn = false
 216         }
 217
 218         if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
 219                 a.exceededAlarmHistoryOn = false
 220         }
 221         a.WriteAlarmInfoToPersistentVolume()
 222
 223         a.mutex.Unlock()
 224         if a.postClear && app.Config.GetBool("controls.noma.enabled") {
 225                 m.PerceivedSeverity = alarm.SeverityCleared
 226                 return a.PostAlarm(m)
 227         }
 228         return nil, nil
 229 }
 230
 231 func timerDelay(delay int) {
 232         timer := time.NewTimer(time.Duration(delay) * time.Second)
 233         <-timer.C
 234 }
 235
 236 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
 237         for i, m := range a.activeAlarms {
 238                 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
 239                         m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
 240                         return i, true
 241                 }
 242         }
 243         return -1, false
 244 }
 245
 246 func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
 247         app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
 248         copy(alarms[i:], alarms[i+1:])
 249         return alarms[:len(alarms)-1]
 250 }
 251
 252 func (a *AlarmManager) GenerateAlarmId() int {
 253         a.uniqueAlarmId++ // @todo: generate a unique ID
 254         return a.uniqueAlarmId
 255 }
 256
 257 func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
 258         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 259         newAlarm.AlarmId = alarmId
 260         newAlarm.AlarmText = alarmDef.AlarmText
 261         newAlarm.EventType = alarmDef.EventType
 262 }
 263
 264 func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
 265         thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
 266         thresholdMessage := alarm.AlarmMessage{
 267                 Alarm:       thresholdAlarm,
 268                 AlarmAction: alarm.AlarmActionRaise,
 269                 AlarmTime:   time.Now().UnixNano(),
 270         }
 271         alarmDef := alarm.RICAlarmDefinitions[sp]
 272         alarmId := a.GenerateAlarmId()
 273         alarmDef.AlarmId = alarmId
 274         a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, *alarmDef})
 275         a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, *alarmDef})
 276
 277         return true
 278 }
 279
 280 func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
 281         /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
 282            The attempt to raise the alarm next time will be suppressed when found as duplicate. */
 283         if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
 284                 app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
 285                 a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
 286         }
 287
 288         // @todo: For now just keep the  active alarms in-memory. Use SDL later for persistence
 289         a.activeAlarms = append(a.activeAlarms, *newAlarm)
 290 }
 291
 292 func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
 293         /* If maximum number of events in alarm history is reached, an error log writing is made,
 294            and new alarm indicating the problem is raised. The attempt to add new event time will
 295            be suppressed */
 296
 297         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 298                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 299                 a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 300         }
 301
 302         // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
 303         a.alarmHistory = append(a.alarmHistory, *newAlarm)
 304 }
 305
 306 func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 307         result, err := json.Marshal(m)
 308         if err != nil {
 309                 app.Logger.Info("json.Marshal failed: %v", err)
 310                 return nil, err
 311         }
 312
 313         fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
 314         app.Logger.Info("Posting alarm to '%s'", fullUrl)
 315
 316         resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
 317         if err != nil || resp == nil {
 318                 app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
 319         }
 320
 321         return nil, err
 322 }
 323
 324 func (a *AlarmManager) GenerateAlertLabels(alarmId int, newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
 325         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 326         amLabels := models.LabelSet{
 327                 "status":      string(status),
 328                 "alertname":   alarmDef.AlarmText,
 329                 "severity":    string(newAlarm.PerceivedSeverity),
 330                 "service":     fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
 331                 "info":        newAlarm.IdentifyingInfo,
 332                 "system_name": "RIC",
 333         }
 334         amAnnotations := models.LabelSet{
 335                 "alarm_id":         fmt.Sprintf("%d", alarmId),
 336                 "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
 337                 "event_type":       alarmDef.EventType,
 338                 "identifying_info": newAlarm.IdentifyingInfo,
 339                 "additional_info":  newAlarm.AdditionalInfo,
 340                 "description":      fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
 341                 "instructions":     alarmDef.OperationInstructions,
 342                 "timestamp":        fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
 343         }
 344
 345         return amLabels, amAnnotations
 346 }
 347
 348 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
 349         cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
 350         return client.New(cr, strfmt.Default)
 351 }
 352
 353 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
 354         pa := &models.PostableAlert{
 355                 Alert: models.Alert{
 356                         GeneratorURL: strfmt.URI(""),
 357                         Labels:       amLabels,
 358                 },
 359                 Annotations: amAnnotations,
 360         }
 361         alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
 362
 363         app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
 364         ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
 365         if err != nil {
 366                 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
 367         }
 368         return ok, err
 369 }
 370
 371 func (a *AlarmManager) StatusCB() bool {
 372         if !a.rmrReady {
 373                 app.Logger.Info("RMR not ready yet!")
 374         }
 375         return a.rmrReady
 376 }
 377
 378 func (a *AlarmManager) ConfigChangeCB(configparam string) {
 379         a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
 380         if a.maxActiveAlarms == 0 {
 381                 a.maxActiveAlarms = 5000
 382         }
 383
 384         a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
 385         if a.maxAlarmHistory == 0 {
 386                 a.maxAlarmHistory = 20000
 387         }
 388
 389         a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 390         a.amHost = viper.GetString("controls.promAlertManager.address")
 391
 392         app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
 393         app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
 394         app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
 395         app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
 396
 397         return
 398 }
 399
 400 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
 401
 402         filename := os.Getenv("DEF_FILE")
 403         file, err := ioutil.ReadFile(filename)
 404         if err == nil {
 405                 data := RicAlarmDefinitions{}
 406                 err = json.Unmarshal([]byte(file), &data)
 407                 if err == nil {
 408                         for _, alarmDefinition := range data.AlarmDefinitions {
 409                                 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
 410                                 if exists {
 411                                         app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
 412                                 } else {
 413                                         app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm  %v", alarmDefinition.AlarmId)
 414                                         ricAlarmDefintion := new(alarm.AlarmDefinition)
 415                                         ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
 416                                         ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
 417                                         ricAlarmDefintion.EventType = alarmDefinition.EventType
 418                                         ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
 419                                         ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
 420                                         ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
 421                                         ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive
 422                                         alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
 423                                 }
 424                         }
 425                 } else {
 426                         app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
 427                 }
 428         } else {
 429                 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
 430         }
 431 }
 432
 433 func (a *AlarmManager) ReadAlarmInfoFromPersistentVolume() {
 434         var alarmpersistentinfo AlarmPersistentInfo
 435         byteValue, rerr := ioutil.ReadFile(a.alarmInfoPvFile)
 436         if rerr != nil {
 437                 app.Logger.Error("ararminfo.json file read error %v", rerr)
 438         } else {
 439                 err := json.Unmarshal(byteValue, &alarmpersistentinfo)
 440                 if err != nil {
 441                         app.Logger.Error("alarmpersistentinfo json unmarshal error %v", err)
 442                 } else {
 443                         a.uniqueAlarmId = alarmpersistentinfo.UniqueAlarmId
 444                         a.activeAlarms = make([]AlarmNotification, len(alarmpersistentinfo.ActiveAlarms))
 445                         a.alarmHistory = make([]AlarmNotification, len(alarmpersistentinfo.AlarmHistory))
 446                         copy(a.activeAlarms, alarmpersistentinfo.ActiveAlarms)
 447                         copy(a.alarmHistory, alarmpersistentinfo.AlarmHistory)
 448                 }
 449         }
 450 }
 451
 452 func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() {
 453         var alarmpersistentinfo AlarmPersistentInfo
 454         alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
 455         alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
 456         alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
 457
 458         copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
 459         copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
 460
 461         wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
 462         if err != nil {
 463                 app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
 464         } else {
 465                 werr := ioutil.WriteFile(a.alarmInfoPvFile, wdata, 0777)
 466                 if werr != nil {
 467                         app.Logger.Error("alarminfo.json file write error %v", werr)
 468                 }
 469         }
 470 }
 471
 472 func (a *AlarmManager) Run(sdlcheck bool, ttlInterval int) {
 473         app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
 474         app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
 475         app.Resource.InjectStatusCb(a.StatusCB)
 476         app.AddConfigChangeListener(a.ConfigChangeCB)
 477
 478         alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
 479         a.ReadAlarmDefinitionFromJson()
 480
 481         a.InjectRoutes()
 482
 483         // Start background timer for re-raising alerts
 484         go a.StartAlertTimer()
 485         go a.StartTTLTimer(ttlInterval)
 486
 487         a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
 488
 489         a.ReadAlarmInfoFromPersistentVolume()
 490
 491         app.RunWithParams(a, sdlcheck)
 492 }
 493
 494 func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
 495         if alertInterval == 0 {
 496                 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 497         }
 498
 499         if amHost == "" {
 500                 amHost = viper.GetString("controls.promAlertManager.address")
 501         }
 502
 503         maxActiveAlarms := app.Config.GetInt("controls.maxActiveAlarms")
 504         if maxActiveAlarms == 0 {
 505                 maxActiveAlarms = 5000
 506         }
 507
 508         maxAlarmHistory := app.Config.GetInt("controls.maxAlarmHistory")
 509         if maxAlarmHistory == 0 {
 510                 maxAlarmHistory = 20000
 511         }
 512
 513         return &AlarmManager{
 514                 rmrReady:               false,
 515                 postClear:              clearAlarm,
 516                 amHost:                 amHost,
 517                 amBaseUrl:              app.Config.GetString("controls.promAlertManager.baseUrl"),
 518                 amSchemes:              []string{app.Config.GetString("controls.promAlertManager.schemes")},
 519                 alertInterval:          alertInterval,
 520                 activeAlarms:           make([]AlarmNotification, 0),
 521                 alarmHistory:           make([]AlarmNotification, 0),
 522                 uniqueAlarmId:          0,
 523                 maxActiveAlarms:        maxActiveAlarms,
 524                 maxAlarmHistory:        maxAlarmHistory,
 525                 exceededActiveAlarmOn:  false,
 526                 exceededAlarmHistoryOn: false,
 527                 alarmInfoPvFile:        app.Config.GetString("controls.alarmInfoPvFile"),
 528         }
 529 }
 530
 531 // Main function
 532 func main() {
 533         NewAlarmManager("", 0, true).Run(true, 10)
 534 }