manager/cmd/manager.go

   1 /*
   2  *  Copyright (c) 2020 AT&T Intellectual Property.
   3  *  Copyright (c) 2020 Nokia.
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  * This source code is part of the near-RT RIC (RAN Intelligent Controller)
  18  * platform project (RICP).
  19  */
  20
  21 package main
  22
  23 import (
  24         "bytes"
  25         "encoding/json"
  26         "fmt"
  27         "io/ioutil"
  28         "net/http"
  29         "os"
  30         "time"
  31
  32         "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
  33         app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
  34         clientruntime "github.com/go-openapi/runtime/client"
  35         "github.com/go-openapi/strfmt"
  36         "github.com/prometheus/alertmanager/api/v2/client"
  37         "github.com/prometheus/alertmanager/api/v2/client/alert"
  38         "github.com/prometheus/alertmanager/api/v2/models"
  39         "github.com/spf13/viper"
  40 )
  41
  42 func (a *AlarmManager) ClearExpiredAlarms(m AlarmNotification, idx int, mLocked bool) bool {
  43         d, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]
  44         if !ok || d.TimeToLive == 0 {
  45                 return false
  46         }
  47
  48         elapsed := (time.Now().UnixNano() - m.AlarmTime) / 1e9
  49         if int(elapsed) >= d.TimeToLive {
  50                 app.Logger.Info("Alarm (sp=%d id=%d) with TTL=%d expired, clearing ...", m.Alarm.SpecificProblem, m.AlarmId, d.TimeToLive)
  51
  52                 m.AlarmAction = alarm.AlarmActionClear
  53                 m.AlarmTime = time.Now().UnixNano()
  54
  55                 if !mLocked { // For testing purpose
  56                         a.mutex.Lock()
  57                 }
  58                 a.ProcessClearAlarm(&m, d, idx)
  59                 return true
  60         }
  61         return false
  62 }
  63
  64 func (a *AlarmManager) StartTTLTimer(interval int) {
  65         tick := time.Tick(time.Duration(interval) * time.Second)
  66         for range tick {
  67                 a.mutex.Lock()
  68                 for idx, m := range a.activeAlarms {
  69                         if a.ClearExpiredAlarms(m, idx, true) {
  70                                 a.mutex.Lock() // ClearExpiredAlarms unlocks the mutex, so re-lock here
  71                                 continue
  72                         }
  73                 }
  74                 a.mutex.Unlock()
  75         }
  76 }
  77
  78 func (a *AlarmManager) StartAlertTimer() {
  79         tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
  80         for range tick {
  81                 a.mutex.Lock()
  82                 for _, m := range a.activeAlarms {
  83                         app.Logger.Info("Re-raising alarm: %v", m)
  84                         a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
  85                 }
  86                 a.mutex.Unlock()
  87         }
  88 }
  89
  90 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
  91         app.Logger.Info("Message received!")
  92
  93         defer app.Rmr.Free(rp.Mbuf)
  94         switch rp.Mtype {
  95         case alarm.RIC_ALARM_UPDATE:
  96                 a.HandleAlarms(rp)
  97         default:
  98                 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
  99         }
 100
 101         return nil
 102 }
 103
 104 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
 105         var m alarm.AlarmMessage
 106         app.Logger.Info("Received JSON: %s", rp.Payload)
 107         if err := json.Unmarshal(rp.Payload, &m); err != nil {
 108                 app.Logger.Error("json.Unmarshal failed: %v", err)
 109                 return nil, err
 110         }
 111         app.Logger.Info("newAlarm: %v", m)
 112
 113         return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
 114 }
 115
 116 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 117         a.mutex.Lock()
 118         alarmDef := &alarm.AlarmDefinition{}
 119         var ok bool
 120         if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
 121                 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
 122                 a.mutex.Unlock()
 123                 return nil, nil
 124         }
 125
 126         idx, found := a.IsMatchFound(m.Alarm)
 127         // Suppress duplicate alarms
 128         if found && m.AlarmAction == alarm.AlarmActionRaise {
 129                 app.Logger.Info("Duplicate alarm found, suppressing ...")
 130                 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
 131                         // Duplicate with same severity found
 132                         a.mutex.Unlock()
 133                         return nil, nil
 134                 } else {
 135                         // Remove duplicate with different severity
 136                         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 137                 }
 138         }
 139
 140         // Clear alarm if found from active alarm list
 141         if found && m.AlarmAction == alarm.AlarmActionClear {
 142                 return a.ProcessClearAlarm(m, alarmDef, idx)
 143         }
 144
 145         // New alarm -> update active alarms and post to Alert Manager
 146         if m.AlarmAction == alarm.AlarmActionRaise {
 147                 return a.ProcessRaiseAlarm(m, alarmDef)
 148         }
 149
 150         a.mutex.Unlock()
 151         return nil, nil
 152 }
 153
 154 func (a *AlarmManager) ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) (*alert.PostAlertsOK, error) {
 155         app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 156
 157         // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
 158         m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
 159         a.UpdateAlarmFields(a.GenerateAlarmId(), m)
 160         a.UpdateActiveAlarmList(m)
 161         a.mutex.Unlock()
 162
 163         if alarmDef.RaiseDelay > 0 {
 164                 timerDelay(alarmDef.RaiseDelay)
 165                 a.mutex.Lock()
 166                 // Alarm may have been deleted from active alarms table during delay or table index may have changed
 167                 idx, found := a.IsMatchFound(m.Alarm)
 168                 if found {
 169                         // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
 170                         a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
 171                         app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
 172                         a.mutex.Unlock()
 173                 } else {
 174                         app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
 175                         a.mutex.Unlock()
 176                         return nil, nil
 177                 }
 178         }
 179
 180         m.AlarmDefinition.RaiseDelay = 0
 181         a.UpdateAlarmHistoryList(m)
 182         a.WriteAlarmInfoToPersistentVolume()
 183
 184         // Send alarm notification to NOMA, if enabled
 185         if app.Config.GetBool("controls.noma.enabled") {
 186                 return a.PostAlarm(m)
 187         }
 188         return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
 189 }
 190
 191 func (a *AlarmManager) ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) (*alert.PostAlertsOK, error) {
 192         app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 193         if alarmDef.ClearDelay > 0 {
 194                 a.mutex.Unlock()
 195                 timerDelay(alarmDef.ClearDelay)
 196                 app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
 197                 a.mutex.Lock()
 198                 // Another alarm clear may have happened during delay and active alarms table index changed
 199                 var found bool
 200                 idx, found = a.IsMatchFound(m.Alarm)
 201                 if !found {
 202                         a.mutex.Unlock()
 203                         return nil, nil
 204                 }
 205         }
 206         a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
 207         a.alarmHistory = append(a.alarmHistory, *m)
 208         a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
 209         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 210                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 211                 a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 212         }
 213
 214         if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
 215                 a.exceededActiveAlarmOn = false
 216         }
 217
 218         if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
 219                 a.exceededAlarmHistoryOn = false
 220         }
 221         a.WriteAlarmInfoToPersistentVolume()
 222
 223         a.mutex.Unlock()
 224         if a.postClear && app.Config.GetBool("controls.noma.enabled") {
 225                 m.PerceivedSeverity = alarm.SeverityCleared
 226                 return a.PostAlarm(m)
 227         }
 228         return nil, nil
 229 }
 230
 231 func timerDelay(delay int) {
 232         timer := time.NewTimer(time.Duration(delay) * time.Second)
 233         <-timer.C
 234 }
 235
 236 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
 237         for i, m := range a.activeAlarms {
 238                 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
 239                         m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
 240                         return i, true
 241                 }
 242         }
 243         return -1, false
 244 }
 245
 246 func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
 247         app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
 248         copy(alarms[i:], alarms[i+1:])
 249         return alarms[:len(alarms)-1]
 250 }
 251
 252 func (a *AlarmManager) GenerateAlarmId() int {
 253         a.uniqueAlarmId++ // @todo: generate a unique ID
 254         return a.uniqueAlarmId
 255 }
 256
 257 func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
 258         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 259         newAlarm.AlarmId = alarmId
 260         newAlarm.AlarmText = alarmDef.AlarmText
 261         newAlarm.EventType = alarmDef.EventType
 262 }
 263
 264 func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
 265         thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
 266         thresholdMessage := alarm.AlarmMessage{
 267                 Alarm:       thresholdAlarm,
 268                 AlarmAction: alarm.AlarmActionRaise,
 269                 AlarmTime:   time.Now().UnixNano(),
 270         }
 271         alarmDef := alarm.RICAlarmDefinitions[sp]
 272         alarmId := a.GenerateAlarmId()
 273         alarmDef.AlarmId = alarmId
 274         a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, *alarmDef})
 275         a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, *alarmDef})
 276
 277         return true
 278 }
 279
 280 func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
 281         /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
 282            The attempt to raise the alarm next time will be suppressed when found as duplicate. */
 283         if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
 284                 app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
 285                 a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
 286         }
 287
 288         // @todo: For now just keep the  active alarms in-memory. Use SDL later for persistence
 289         a.activeAlarms = append(a.activeAlarms, *newAlarm)
 290 }
 291
 292 func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
 293         /* If maximum number of events in alarm history is reached, an error log writing is made,
 294            and new alarm indicating the problem is raised. The attempt to add new event time will
 295            be suppressed */
 296
 297         if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
 298                 app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
 299                 a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
 300         }
 301
 302         // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
 303         a.alarmHistory = append(a.alarmHistory, *newAlarm)
 304 }
 305
 306 func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
 307         result, err := json.Marshal(m)
 308         if err != nil {
 309                 app.Logger.Info("json.Marshal failed: %v", err)
 310                 return nil, err
 311         }
 312
 313         fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
 314         app.Logger.Info("Posting alarm to '%s'", fullUrl)
 315
 316         resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
 317         if err != nil || resp == nil {
 318                 app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
 319         }
 320
 321         return nil, err
 322 }
 323
 324 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
 325         alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
 326         amLabels := models.LabelSet{
 327                 "status":      string(status),
 328                 "alertname":   alarmDef.AlarmText,
 329                 "severity":    string(newAlarm.PerceivedSeverity),
 330                 "service":     fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
 331                 "system_name": "RIC",
 332         }
 333         amAnnotations := models.LabelSet{
 334                 "alarm_id":         fmt.Sprintf("%d", alarmDef.AlarmId),
 335                 "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
 336                 "event_type":       alarmDef.EventType,
 337                 "identifying_info": newAlarm.IdentifyingInfo,
 338                 "additional_info":  newAlarm.AdditionalInfo,
 339                 "description":      fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
 340                 "instructions":     alarmDef.OperationInstructions,
 341                 "timestamp":        fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
 342         }
 343
 344         return amLabels, amAnnotations
 345 }
 346
 347 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
 348         cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
 349         return client.New(cr, strfmt.Default)
 350 }
 351
 352 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
 353         pa := &models.PostableAlert{
 354                 Alert: models.Alert{
 355                         GeneratorURL: strfmt.URI(""),
 356                         Labels:       amLabels,
 357                 },
 358                 Annotations: amAnnotations,
 359         }
 360         alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
 361
 362         app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
 363         ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
 364         if err != nil {
 365                 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
 366         }
 367         return ok, err
 368 }
 369
 370 func (a *AlarmManager) StatusCB() bool {
 371         if !a.rmrReady {
 372                 app.Logger.Info("RMR not ready yet!")
 373         }
 374
 375         return a.rmrReady
 376 }
 377
 378 func (a *AlarmManager) ConfigChangeCB(configparam string) {
 379
 380         a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
 381         a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
 382         a.alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 383         a.amHost = viper.GetString("controls.promAlertManager.address")
 384
 385         app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
 386         app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
 387         app.Logger.Debug("ConfigChangeCB: alertInterval %v", a.alertInterval)
 388         app.Logger.Debug("ConfigChangeCB: amHost = %v", a.amHost)
 389
 390         return
 391 }
 392
 393 func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
 394
 395         filename := os.Getenv("DEF_FILE")
 396         file, err := ioutil.ReadFile(filename)
 397         if err == nil {
 398                 data := RicAlarmDefinitions{}
 399                 err = json.Unmarshal([]byte(file), &data)
 400                 if err == nil {
 401                         for _, alarmDefinition := range data.AlarmDefinitions {
 402                                 _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
 403                                 if exists {
 404                                         app.Logger.Error("ReadAlarmDefinitionFromJson: alarm definition already exists for %v", alarmDefinition.AlarmId)
 405                                 } else {
 406                                         app.Logger.Debug("ReadAlarmDefinitionFromJson: alarm  %v", alarmDefinition.AlarmId)
 407                                         ricAlarmDefintion := new(alarm.AlarmDefinition)
 408                                         ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
 409                                         ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
 410                                         ricAlarmDefintion.EventType = alarmDefinition.EventType
 411                                         ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
 412                                         ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
 413                                         ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
 414                                         ricAlarmDefintion.TimeToLive = alarmDefinition.TimeToLive
 415                                         alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
 416                                 }
 417                         }
 418                 } else {
 419                         app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
 420                 }
 421         } else {
 422                 app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
 423         }
 424 }
 425
 426 func (a *AlarmManager) ReadAlarmInfoFromPersistentVolume() {
 427         var alarmpersistentinfo AlarmPersistentInfo
 428         byteValue, rerr := ioutil.ReadFile(a.alarmInfoPvFile)
 429         if rerr != nil {
 430                 app.Logger.Error("ararminfo.json file read error %v", rerr)
 431         } else {
 432                 err := json.Unmarshal(byteValue, &alarmpersistentinfo)
 433                 if err != nil {
 434                         app.Logger.Error("alarmpersistentinfo json unmarshal error %v", err)
 435                 } else {
 436                         a.uniqueAlarmId = alarmpersistentinfo.UniqueAlarmId
 437                         a.activeAlarms = make([]AlarmNotification, len(alarmpersistentinfo.ActiveAlarms))
 438                         a.alarmHistory = make([]AlarmNotification, len(alarmpersistentinfo.AlarmHistory))
 439                         copy(a.activeAlarms, alarmpersistentinfo.ActiveAlarms)
 440                         copy(a.alarmHistory, alarmpersistentinfo.AlarmHistory)
 441                 }
 442         }
 443 }
 444
 445 func (a *AlarmManager) WriteAlarmInfoToPersistentVolume() {
 446         var alarmpersistentinfo AlarmPersistentInfo
 447         alarmpersistentinfo.UniqueAlarmId = a.uniqueAlarmId
 448         alarmpersistentinfo.ActiveAlarms = make([]AlarmNotification, len(a.activeAlarms))
 449         alarmpersistentinfo.AlarmHistory = make([]AlarmNotification, len(a.alarmHistory))
 450
 451         copy(alarmpersistentinfo.ActiveAlarms, a.activeAlarms)
 452         copy(alarmpersistentinfo.AlarmHistory, a.alarmHistory)
 453
 454         wdata, err := json.MarshalIndent(alarmpersistentinfo, "", " ")
 455         if err != nil {
 456                 app.Logger.Error("alarmpersistentinfo json marshal error %v", err)
 457         } else {
 458                 werr := ioutil.WriteFile(a.alarmInfoPvFile, wdata, 0777)
 459                 if werr != nil {
 460                         app.Logger.Error("alarminfo.json file write error %v", werr)
 461                 }
 462         }
 463 }
 464
 465 func (a *AlarmManager) Run(sdlcheck bool, ttlInterval int) {
 466         app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
 467         app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
 468         app.Resource.InjectStatusCb(a.StatusCB)
 469         app.AddConfigChangeListener(a.ConfigChangeCB)
 470
 471         alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
 472         a.ReadAlarmDefinitionFromJson()
 473
 474         a.InjectRoutes()
 475
 476         // Start background timer for re-raising alerts
 477         go a.StartAlertTimer()
 478         go a.StartTTLTimer(ttlInterval)
 479
 480         a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
 481
 482         a.ReadAlarmInfoFromPersistentVolume()
 483
 484         app.RunWithParams(a, sdlcheck)
 485 }
 486
 487 func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
 488         if alertInterval == 0 {
 489                 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
 490         }
 491
 492         if amHost == "" {
 493                 amHost = viper.GetString("controls.promAlertManager.address")
 494         }
 495
 496         return &AlarmManager{
 497                 rmrReady:               false,
 498                 postClear:              clearAlarm,
 499                 amHost:                 amHost,
 500                 amBaseUrl:              app.Config.GetString("controls.promAlertManager.baseUrl"),
 501                 amSchemes:              []string{app.Config.GetString("controls.promAlertManager.schemes")},
 502                 alertInterval:          alertInterval,
 503                 activeAlarms:           make([]AlarmNotification, 0),
 504                 alarmHistory:           make([]AlarmNotification, 0),
 505                 uniqueAlarmId:          0,
 506                 maxActiveAlarms:        app.Config.GetInt("controls.maxActiveAlarms"),
 507                 maxAlarmHistory:        app.Config.GetInt("controls.maxAlarmHistory"),
 508                 exceededActiveAlarmOn:  false,
 509                 exceededAlarmHistoryOn: false,
 510                 alarmInfoPvFile:        app.Config.GetString("controls.alarmInfoPvFile"),
 511         }
 512 }
 513
 514 // Main function
 515 func main() {
 516         NewAlarmManager("", 0, true).Run(true, 10)
 517 }