manager/cmd/manager.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io/ioutil"
  28         "net/http"
  29         "os"
  30         "time"
  31
  32         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  33         app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  34         clientruntime "github.com/go-openapi/runtime/client"
  35         "github.com/go-openapi/strfmt"
  36         "github.com/prometheus/alertmanager/api/v2/client"
  37         "github.com/prometheus/alertmanager/api/v2/client/alert"
  38         "github.com/prometheus/alertmanager/api/v2/models"
  39         "github.com/spf13/viper"
  40 )
  41
  42 func (a *AlarmManager) ClearExpiredAlarms(m AlarmNotification, idx int, mLocked bool) bool {
  43         d, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]
  44         if !ok || d.TimeToLive == 0 {
  45                 return false
  46         }
  47
  48         elapsed := (time.Now().UnixNano() - m.AlarmTime) / 1e9
  49         if int(elapsed) >= d.TimeToLive {
  50                 app.Logger.Info("Alarm (sp=%d id=%d) with TTL=%d expired, clearing ...", m.Alarm.SpecificProblem, m.AlarmId, d.TimeToLive)
  51
  52                 m.AlarmAction = alarm.AlarmActionClear
  53                 m.AlarmTime = time.Now().UnixNano()
  54
  55                 if !mLocked { // For testing purpose
  56                         a.mutex.Lock()
  57                 }
  58                 a.ProcessClearAlarm(&m, d, idx)
  59                 return true
  60         }
  61         return false
  62 }
  63
  64 func (a *AlarmManager) StartTTLTimer(interval int) {
  65         tick := time.Tick(time.Duration(interval) * time.Second)
  66         for range tick {
  67                 a.mutex.Lock()
  68                 for idx, m := range a.activeAlarms {
  69                         if a.ClearExpiredAlarms(m, idx, true) {
  70                                 a.mutex.Lock() // ClearExpiredAlarms unlocks the mutex, so re-lock here
  71                                 continue
  72                         }
  73                 }
  74                 a.mutex.Unlock()
  75         }
  76 }
  77
  78 func (a *AlarmManager) StartAlertTimer() {
  79         tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
  80         for range tick {
  81                 a.mutex.Lock()
  82                 for _, m := range a.activeAlarms {
  83                         app.Logger.Info("Re-raising alarm: %v", m)
  84                         a.PostAlert(a.GenerateAlertLabels(m.AlarmId, m.Alarm, AlertStatusActive, m.AlarmTime))
  85                 }
  86                 a.mutex.Unlock()
  87         }
  88 }
  89
  90 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
  91         app.Logger.Info("Message received!")
  92
  93         defer app.Rmr.Free(rp.Mbuf)
  94         switch rp.Mtype {
  95         case alarm.RIC_ALARM_UPDATE:
  96                 a.HandleAlarms(rp)
  97         default:
  98                 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
  99         }
 100
 101         return nil
 102 }
 103
 104 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
 105         var m alarm.AlarmMessage
 106         app.Logger.Info("Received JSON: %s", rp.Payload)
 107         if err := json.Unmarshal(rp.Payload, &m); err != nil {
 108                 app.Logger.Error("json.Unmarshal failed: %v", err)
 109                 return nil, err
 110         }
 111         app.Logger.Info("newAlarm: %v", m)
 112
 113         return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
 114 }
 115
 116 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 117         a.mutex.Lock()
 118         alarmDef := &alarm.AlarmDefinition{}
 119         var ok bool
 120         if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
 121                 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
 122                 a.mutex.Unlock()
 123                 return nil, nil
 124         }
 125
 126         idx, found := a.IsMatchFound(m.Alarm)
 127         // Suppress duplicate alarms
 128         if found && m.AlarmAction == alarm.AlarmActionRaise {
 129                 app.Logger.Info("Duplicate alarm found, suppressing ...")
 130                 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
 131                         // Duplicate with same severity found
 132                         a.mutex.Unlock()
 133                         return nil, nil
 134                 } else {
 135                         // Remove duplicate with different severity
 136                         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 137                 }
 138         }
 139
 140         // Clear alarm if found from active alarm list
 141         if found && m.AlarmAction == alarm.AlarmActionClear {
 142                 return a.ProcessClearAlarm(m, alarmDef, idx)
 143         }
 144
 145         // New alarm -> update active alarms and post to Alert Manager
 146         if m.AlarmAction == alarm.AlarmActionRaise {
 147                 return a.ProcessRaiseAlarm(m, alarmDef)
 148         }
 149
 150         a.mutex.Unlock()
 151         return nil, nil
 152 }
 153
 154 func (a *AlarmManager) ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) (*alert.PostAlertsOK, error) {
 155         app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 156
 157         // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
 158         m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
 159         a.UpdateAlarmFields(a.GenerateAlarmId(), m)
 160         a.UpdateActiveAlarmList(m)
 161         a.mutex.Unlock()
 162
 163         if alarmDef.RaiseDelay > 0 {
 164                 timerDelay(alarmDef.RaiseDelay)
 165                 a.mutex.Lock()
 166                 // Alarm may have been deleted from active alarms table during delay or table index may have changed
 167                 idx, found := a.IsMatchFound(m.Alarm)
 168                 if found {
 169                         // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
 170                         a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
 171                         app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 172                         a.mutex.Unlock()
 173                 } else {
 174                         app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
 175                         a.mutex.Unlock()
 176                         return nil, nil
 177                 }
 178         }
 179
 180         m.AlarmDefinition.RaiseDelay = 0
 181         a.UpdateAlarmHistoryList(m)
 182         a.WriteAlarmInfoToPersistentVolume()
 183
 184         // Send alarm notification to NOMA, if enabled
 185         if app.Config.GetBool("controls.noma.enabled") {
 186                 return a.PostAlarm(m)
 187         }
 188         return a.PostAlert(a.GenerateAlertLabels(m.AlarmId, m.Alarm, AlertStatusActive, m.AlarmTime))
 189 }
 190
 191 func (a *AlarmManager) ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) (*alert.PostAlertsOK, error) {
 192         app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 193         if alarmDef.ClearDelay > 0 {
 194                 a.mutex.Unlock()
 195                 timerDelay(alarmDef.ClearDelay)
 196                 app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 197                 a.mutex.Lock()
 198                 // Another alarm clear may have happened during delay and active alarms table index changed
 199                 var found bool
 200                 idx, found = a.IsMatchFound(m.Alarm)
 201                 if !found {
 202                         a.mutex.Unlock()
 203                         return nil, nil
 204                 }
 205         }
 206         a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
 207         a.alarmHistory = append(a.alarmHistory, *m)
 208         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 209         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 210                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 211                 a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 212         }
 213
 214         if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
 215                 a.exceededActiveAlarmOn = false
 216         }
 217
 218         if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
 219                 a.exceededAlarmHistoryOn = false
 220         }
 221         a.WriteAlarmInfoToPersistentVolume()
 222
 223         a.mutex.Unlock()
 224         if a.postClear && app.Config.GetBool("controls.noma.enabled") {
 225                 m.PerceivedSeverity = alarm.SeverityCleared
 226                 return a.PostAlarm(m)
 227         }
 228         return nil, nil
 229 }
 230
 231 func timerDelay(delay int) {
 232         timer := time.NewTimer(time.Duration(delay) * time.Second)
 233         <-timer.C
 234 }
 235
 236 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
 237         for i, m := range a.activeAlarms {
 238                 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
 239                         m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
 240                         return i, true
 241                 }
 242         }
 243         return -1, false
 244 }
 245
 246 func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
 247         app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
 248         copy(alarms[i:], alarms[i+1:])
 249         return alarms[:len(alarms)-1]
 250 }
 251
 252 func (a *AlarmManager) GenerateAlarmId() int {
 253         a.uniqueAlarmId++ // @todo: generate a unique ID
 254         return a.uniqueAlarmId
 255 }
 256
 257 func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
 258         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 259         newAlarm.AlarmId = alarmId
 260         newAlarm.AlarmText = alarmDef.AlarmText
 261         newAlarm.EventType = alarmDef.EventType
 262 }
 263
 264 func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
 265         thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
 266         thresholdMessage := alarm.AlarmMessage{
 267                 Alarm:       thresholdAlarm,
 268                 AlarmAction: alarm.AlarmActionRaise,
 269                 AlarmTime:   time.Now().UnixNano(),
 270         }
 271         alarmDef := alarm.RICAlarmDefinitions[sp]
 272         alarmId := a.GenerateAlarmId()
 273         alarmDef.AlarmId = alarmId
 274         a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, *alarmDef})
 275         a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, *alarmDef})
 276
 277         return true
 278 }
 279
 280 func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
 281         /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
 282            The attempt to raise the alarm next time will be suppressed when found as duplicate. */
 283         if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
 284                 app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
 285                 a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
 286         }
 287
 288         // @todo: For now just keep the  active alarms in-memory. Use SDL later for persistence
 289         a.activeAlarms = append(a.activeAlarms, *newAlarm)
 290 }
 291
 292 func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
 293         /* If maximum number of events in alarm history is reached, an error log writing is made,
 294            and new alarm indicating the problem is raised. The attempt to add new event time will
 295            be suppressed */
 296
 297         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 298                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 299                 a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 300         }
 301
 302         // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
 303         a.alarmHistory = append(a.alarmHistory, *newAlarm)
 304 }
 305
 306 func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 307         result, err := json.Marshal(m)
 308         if err != nil {
 309                 app.Logger.Info("json.Marshal failed: %v", err)
 310                 return nil, err
 311         }
 312
 313         fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
 314         app.Logger.Info("Posting alarm to '%s'", fullUrl)
 315
 316         resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
 317         if err != nil || resp == nil {
 318                 app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
 319         }
 320
 321         return nil, err
 322 }
 323
 324 func (a *AlarmManager) GenerateAlertLabels(alarmId int, newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
 325         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 326         amLabels := models.LabelSet{
 327                 "status":      string(status),
 328                 "alertname":   alarmDef.AlarmText,
 329                 "severity":    string(newAlarm.PerceivedSeverity),
 330                 "service":     fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
 331                 "system_name": "RIC",
 332         }
 333         amAnnotations := models.LabelSet{
 334                 "alarm_id":         fmt.Sprintf("%d", alarmId),
 335                 "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
 336                 "event_type":       alarmDef.EventType,
 337                 "identifying_info": newAlarm.IdentifyingInfo,
 338                 "additional_info":  newAlarm.AdditionalInfo,
 339                 "description":      fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
 340                 "instructions":     alarmDef.OperationInstructions,
 341                 "timestamp":        fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
 342         }
 343
 344         return amLabels, amAnnotations
 345 }
 346
 347 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
 348         cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
 349         return client.New(cr, strfmt.Default)
 350 }
 351
 352 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
 353         pa := &models.PostableAlert{
 354                 Alert: models.Alert{
 355                         GeneratorURL: strfmt.URI(""),
 356                         Labels:       amLabels,
 357                 },
 358                 Annotations: amAnnotations,
 359         }
 360         alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
 361
 362         app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
 363         ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
 364         if err != nil {
 365                 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
 366         }
 367         return ok, err
 368 }
 369
 370 func (a *AlarmManager) StatusCB() bool {
 371         if !a.rmrReady {
 372                 app.Logger.Info("RMR not ready yet!")
 373         }
 374
 375         return a.rmrReady
 376 }
 377
 378 func (a *AlarmManager) ConfigChangeCB(configparam string) {
 379
 380         a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
 381         if a.maxActiveAlarms == 0 {
 382                 a.maxActiveAlarms = 5000
 383         }
 384
 385         a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
 386         if a.maxAlarmHistory == 0 {
 387                 a.maxAlarmHistory = 20000
 388         }
 389
 390         a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 391         a.amHost = viper.GetString("controls.promAlertManager.address")
 392
 393         app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
 394         app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
 395         app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
 396         app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
 397
 398         return
 399 }
 400
 401 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
 402
 403         filename := os.Getenv("DEF_FILE")
 404         file, err := ioutil.ReadFile(filename)
 405         if err == nil {
 406                 data := RicAlarmDefinitions{}
 407                 err = json.Unmarshal([]byte(file), &data)
 408                 if err == nil {
 409                         for _, alarmDefinition := range data.AlarmDefinitions {
 410                                 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
 411                                 if exists {
 412                                         app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
 413                                 } else {
 414                                         app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm  %v", alarmDefinition.AlarmId)
 415                                         ricAlarmDefintion := new(alarm.AlarmDefinition)
 416                                         ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
 417                                         ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
 418                                         ricAlarmDefintion.EventType = alarmDefinition.EventType
 419                                         ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
 420                                         ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
 421                                         ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
 422                                         ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive
 423                                         alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
 424                                 }
 425                         }
 426                 } else {
 427                         app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
 428                 }
 429         } else {
 430                 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
 431         }
 432 }
 433
 434 func (a *AlarmManager) ReadAlarmInfoFromPersistentVolume() {
 435         var alarmpersistentinfo AlarmPersistentInfo
 436         byteValue, rerr := ioutil.ReadFile(a.alarmInfoPvFile)
 437         if rerr != nil {
 438                 app.Logger.Error("ararminfo.json file read error %v", rerr)
 439         } else {
 440                 err := json.Unmarshal(byteValue, &alarmpersistentinfo)
 441                 if err != nil {
 442                         app.Logger.Error("alarmpersistentinfo json unmarshal error %v", err)
 443                 } else {
 444                         a.uniqueAlarmId = alarmpersistentinfo.UniqueAlarmId
 445                         a.activeAlarms = make([]AlarmNotification, len(alarmpersistentinfo.ActiveAlarms))
 446                         a.alarmHistory = make([]AlarmNotification, len(alarmpersistentinfo.AlarmHistory))
 447                         copy(a.activeAlarms, alarmpersistentinfo.ActiveAlarms)
 448                         copy(a.alarmHistory, alarmpersistentinfo.AlarmHistory)
 449                 }
 450         }
 451 }
 452
 453 func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() {
 454         var alarmpersistentinfo AlarmPersistentInfo
 455         alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
 456         alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
 457         alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
 458
 459         copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
 460         copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
 461
 462         wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
 463         if err != nil {
 464                 app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
 465         } else {
 466                 werr := ioutil.WriteFile(a.alarmInfoPvFile, wdata, 0777)
 467                 if werr != nil {
 468                         app.Logger.Error("alarminfo.json file write error %v", werr)
 469                 }
 470         }
 471 }
 472
 473 func (a *AlarmManager) Run(sdlcheck bool, ttlInterval int) {
 474         app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
 475         app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
 476         app.Resource.InjectStatusCb(a.StatusCB)
 477         app.AddConfigChangeListener(a.ConfigChangeCB)
 478
 479         alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
 480         a.ReadAlarmDefinitionFromJson()
 481
 482         a.InjectRoutes()
 483
 484         // Start background timer for re-raising alerts
 485         go a.StartAlertTimer()
 486         go a.StartTTLTimer(ttlInterval)
 487
 488         a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
 489
 490         a.ReadAlarmInfoFromPersistentVolume()
 491
 492         app.RunWithParams(a, sdlcheck)
 493 }
 494
 495 func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
 496         if alertInterval == 0 {
 497                 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 498         }
 499
 500         if amHost == "" {
 501                 amHost = viper.GetString("controls.promAlertManager.address")
 502         }
 503
 504         maxActiveAlarms := app.Config.GetInt("controls.maxActiveAlarms")
 505         if maxActiveAlarms == 0 {
 506                 maxActiveAlarms = 5000
 507         }
 508
 509         maxAlarmHistory := app.Config.GetInt("controls.maxAlarmHistory")
 510         if maxAlarmHistory == 0 {
 511                 maxAlarmHistory = 20000
 512         }
 513
 514         return &AlarmManager{
 515                 rmrReady:               false,
 516                 postClear:              clearAlarm,
 517                 amHost:                 amHost,
 518                 amBaseUrl:              app.Config.GetString("controls.promAlertManager.baseUrl"),
 519                 amSchemes:              []string{app.Config.GetString("controls.promAlertManager.schemes")},
 520                 alertInterval:          alertInterval,
 521                 activeAlarms:           make([]AlarmNotification, 0),
 522                 alarmHistory:           make([]AlarmNotification, 0),
 523                 uniqueAlarmId:          0,
 524                 maxActiveAlarms:        maxActiveAlarms,
 525                 maxAlarmHistory:        maxAlarmHistory,
 526                 exceededActiveAlarmOn:  false,
 527                 exceededAlarmHistoryOn: false,
 528                 alarmInfoPvFile:        app.Config.GetString("controls.alarmInfoPvFile"),
 529         }
 530 }
 531
 532 // Main function
 533 func main() {
 534         NewAlarmManager("", 0, true).Run(true, 10)
 535 }