X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=dmaap-mediator-producer%2Finternal%2Fjobs%2Fjobs.go;h=6dad5fd9769f8da8adca72d5dabbca518ad76c8e;hb=46a0fd717e5f49ebae6cb2c4fbcf54f0e329dc86;hp=eec59c39046175ed569bf1df2609a28b28c640c6;hpb=280385634f160bb78a8944998e9106a2f6549eb0;p=nonrtric.git diff --git a/dmaap-mediator-producer/internal/jobs/jobs.go b/dmaap-mediator-producer/internal/jobs/jobs.go index eec59c39..6dad5fd9 100644 --- a/dmaap-mediator-producer/internal/jobs/jobs.go +++ b/dmaap-mediator-producer/internal/jobs/jobs.go @@ -24,65 +24,77 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" - "strings" "sync" log "github.com/sirupsen/logrus" + "oransc.org/nonrtric/dmaapmediatorproducer/internal/config" "oransc.org/nonrtric/dmaapmediatorproducer/internal/restclient" ) -type Type struct { - TypeId string `json:"id"` - DMaaPTopic string `json:"dmaapTopic"` - Schema string `json:"schema"` - Jobs map[string]JobInfo +type TypeData struct { + TypeId string `json:"id"` + DMaaPTopicURL string `json:"dmaapTopicUrl"` + jobsHandler *jobsHandler } type JobInfo struct { - Owner string `json:"owner"` - LastUpdated string `json:"last_updated"` - InfoJobIdentity string `json:"info_job_identity"` - TargetUri string `json:"target_uri"` - InfoJobData string `json:"info_job_data"` - InfoTypeIdentity string `json:"info_type_identity"` + Owner string `json:"owner"` + LastUpdated string `json:"last_updated"` + InfoJobIdentity string `json:"info_job_identity"` + TargetUri string `json:"target_uri"` + InfoJobData interface{} `json:"info_job_data"` + InfoTypeIdentity string `json:"info_type_identity"` } -type JobHandler interface { - AddJob(JobInfo) error +type JobTypesManager interface { + LoadTypesFromConfiguration() ([]config.TypeDefinition, error) + GetSupportedTypes() []string } -var ( - mu sync.Mutex - typeDir = "configs" - Handler JobHandler - allJobs = make(map[string]Type) -) - -func init() { - Handler = newJobHandlerImpl() +type JobsManager interface { + AddJobFromRESTCall(JobInfo) error + DeleteJobFromRESTCall(jobId string) } -type jobHandlerImpl struct{} +type JobsManagerImpl struct { + configFile string + allTypes map[string]TypeData + pollClient restclient.HTTPClient + mrAddress string + distributeClient restclient.HTTPClient +} -func newJobHandlerImpl() *jobHandlerImpl { - return &jobHandlerImpl{} +func NewJobsManagerImpl(typeConfigFilePath string, pollClient restclient.HTTPClient, mrAddr string, distributeClient restclient.HTTPClient) *JobsManagerImpl { + return &JobsManagerImpl{ + configFile: typeConfigFilePath, + allTypes: make(map[string]TypeData), + pollClient: pollClient, + mrAddress: mrAddr, + distributeClient: distributeClient, + } } -func (jh *jobHandlerImpl) AddJob(ji JobInfo) error { - mu.Lock() - defer mu.Unlock() - if err := validateJobInfo(ji); err == nil { - jobs := allJobs[ji.InfoTypeIdentity].Jobs - jobs[ji.InfoJobIdentity] = ji +func (jm *JobsManagerImpl) AddJobFromRESTCall(ji JobInfo) error { + if err := jm.validateJobInfo(ji); err == nil { + typeData := jm.allTypes[ji.InfoTypeIdentity] + typeData.jobsHandler.addJobCh <- ji + log.Debug("Added job: ", ji) return nil } else { return err } } -func validateJobInfo(ji JobInfo) error { - if _, ok := allJobs[ji.InfoTypeIdentity]; !ok { +func (jm *JobsManagerImpl) DeleteJobFromRESTCall(jobId string) { + for _, typeData := range jm.allTypes { + log.Debugf("Deleting job %v from type %v", jobId, typeData.TypeId) + typeData.jobsHandler.deleteJobCh <- jobId + } + log.Debug("Deleted job: ", jobId) +} + +func (jm *JobsManagerImpl) validateJobInfo(ji JobInfo) error { + if _, ok := jm.allTypes[ji.InfoTypeIdentity]; !ok { return fmt.Errorf("type not supported: %v", ji.InfoTypeIdentity) } if ji.InfoJobIdentity == "" { @@ -95,100 +107,179 @@ func validateJobInfo(ji JobInfo) error { return nil } -func GetTypes() ([]*Type, error) { - mu.Lock() - defer mu.Unlock() - types := make([]*Type, 0, 1) - err := filepath.Walk(typeDir, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if strings.Contains(path, ".json") { - if jobType, err := getType(path); err == nil { - types = append(types, jobType) - } - } - return nil - }) +func (jm *JobsManagerImpl) LoadTypesFromConfiguration() ([]config.TypeDefinition, error) { + typeDefsByte, err := os.ReadFile(jm.configFile) + if err != nil { + return nil, err + } + typeDefs := struct { + Types []config.TypeDefinition `json:"types"` + }{} + err = json.Unmarshal(typeDefsByte, &typeDefs) if err != nil { return nil, err } - return types, nil + for _, typeDef := range typeDefs.Types { + jm.allTypes[typeDef.Id] = TypeData{ + TypeId: typeDef.Id, + DMaaPTopicURL: typeDef.DmaapTopicURL, + jobsHandler: newJobsHandler(typeDef.Id, typeDef.DmaapTopicURL, jm.pollClient, jm.distributeClient), + } + } + return typeDefs.Types, nil } -func GetSupportedTypes() []string { - mu.Lock() - defer mu.Unlock() +func (jm *JobsManagerImpl) GetSupportedTypes() []string { supportedTypes := []string{} - for k := range allJobs { + for k := range jm.allTypes { supportedTypes = append(supportedTypes, k) } return supportedTypes } -func AddJob(job JobInfo) error { - return Handler.AddJob(job) +func (jm *JobsManagerImpl) StartJobsForAllTypes() { + for _, jobType := range jm.allTypes { + + go jobType.jobsHandler.startPollingAndDistribution(jm.mrAddress) + + } } -func getType(path string) (*Type, error) { - if typeDefinition, err := os.ReadFile(path); err == nil { - var dat map[string]interface{} - if marshalError := json.Unmarshal(typeDefinition, &dat); marshalError == nil { - schema, _ := json.Marshal(dat["schema"]) - typeInfo := Type{ - TypeId: dat["id"].(string), - DMaaPTopic: dat["dmaapTopic"].(string), - Schema: string(schema), - Jobs: make(map[string]JobInfo), - } - if _, ok := allJobs[typeInfo.TypeId]; !ok { - allJobs[typeInfo.TypeId] = typeInfo +type jobsHandler struct { + mu sync.Mutex + typeId string + topicUrl string + jobs map[string]job + addJobCh chan JobInfo + deleteJobCh chan string + pollClient restclient.HTTPClient + distributeClient restclient.HTTPClient +} + +func newJobsHandler(typeId string, topicURL string, pollClient restclient.HTTPClient, distributeClient restclient.HTTPClient) *jobsHandler { + return &jobsHandler{ + typeId: typeId, + topicUrl: topicURL, + jobs: make(map[string]job), + addJobCh: make(chan JobInfo), + deleteJobCh: make(chan string), + pollClient: pollClient, + distributeClient: distributeClient, + } +} + +func (jh *jobsHandler) startPollingAndDistribution(mRAddress string) { + go func() { + for { + jh.pollAndDistributeMessages(mRAddress) + } + }() + + go func() { + for { + jh.monitorManagementChannels() + } + }() +} + +func (jh *jobsHandler) pollAndDistributeMessages(mRAddress string) { + log.Debugf("Processing jobs for type: %v", jh.typeId) + messagesBody, error := restclient.Get(mRAddress+jh.topicUrl, jh.pollClient) + if error != nil { + log.Warn("Error getting data from MR. Cause: ", error) + } + log.Debug("Received messages: ", string(messagesBody)) + jh.distributeMessages(messagesBody) +} + +func (jh *jobsHandler) distributeMessages(messages []byte) { + if len(messages) > 2 { + jh.mu.Lock() + defer jh.mu.Unlock() + for _, job := range jh.jobs { + if len(job.messagesChannel) < cap(job.messagesChannel) { + job.messagesChannel <- messages + } else { + jh.emptyMessagesBuffer(job) } - return &typeInfo, nil - } else { - return nil, marshalError } - } else { - return nil, err } } -func RunJobs(mRAddress string) { +func (jh *jobsHandler) emptyMessagesBuffer(job job) { + log.Debug("Emptying message queue for job: ", job.jobInfo.InfoJobIdentity) +out: for { - pollAndDistributeMessages(mRAddress) + select { + case <-job.messagesChannel: + default: + break out + } } } -func pollAndDistributeMessages(mRAddress string) { - for typeId, typeInfo := range allJobs { - log.Debugf("Processing jobs for type: %v", typeId) - messagesBody, error := restclient.Get(fmt.Sprintf("%v/events/%v/users/dmaapmediatorproducer", mRAddress, typeInfo.DMaaPTopic)) - if error != nil { - log.Warnf("Error getting data from MR. Cause: %v", error) - continue - } - distributeMessages(messagesBody, typeInfo) +func (jh *jobsHandler) monitorManagementChannels() { + select { + case addedJob := <-jh.addJobCh: + jh.addJob(addedJob) + case deletedJob := <-jh.deleteJobCh: + jh.deleteJob(deletedJob) } } -func distributeMessages(messages []byte, typeInfo Type) { - if len(messages) > 2 { - mu.Lock() - for _, jobInfo := range typeInfo.Jobs { - go sendMessagesToConsumer(messages, jobInfo) - } - mu.Unlock() +func (jh *jobsHandler) addJob(addedJob JobInfo) { + jh.mu.Lock() + log.Debug("Add job: ", addedJob) + newJob := newJob(addedJob, jh.distributeClient) + go newJob.start() + jh.jobs[addedJob.InfoJobIdentity] = newJob + jh.mu.Unlock() +} + +func (jh *jobsHandler) deleteJob(deletedJob string) { + jh.mu.Lock() + log.Debug("Delete job: ", deletedJob) + j, exist := jh.jobs[deletedJob] + if exist { + j.controlChannel <- struct{}{} + delete(jh.jobs, deletedJob) + } + jh.mu.Unlock() +} + +type job struct { + jobInfo JobInfo + client restclient.HTTPClient + messagesChannel chan []byte + controlChannel chan struct{} +} + +func newJob(j JobInfo, c restclient.HTTPClient) job { + return job{ + jobInfo: j, + client: c, + messagesChannel: make(chan []byte, 10), + controlChannel: make(chan struct{}), } } -func sendMessagesToConsumer(messages []byte, jobInfo JobInfo) { - log.Debugf("Processing job: %v", jobInfo.InfoJobIdentity) - if postErr := restclient.Post(jobInfo.TargetUri, messages); postErr != nil { - log.Warnf("Error posting data for job: %v. Cause: %v", jobInfo, postErr) +func (j *job) start() { +out: + for { + select { + case <-j.controlChannel: + log.Debug("Stop distribution for job: ", j.jobInfo.InfoJobIdentity) + break out + case msg := <-j.messagesChannel: + j.sendMessagesToConsumer(msg) + } } } -func clearAll() { - allJobs = make(map[string]Type) +func (j *job) sendMessagesToConsumer(messages []byte) { + log.Debug("Processing job: ", j.jobInfo.InfoJobIdentity) + if postErr := restclient.Post(j.jobInfo.TargetUri, messages, j.client); postErr != nil { + log.Warnf("Error posting data for job: %v. Cause: %v", j.jobInfo, postErr) + } + log.Debugf("Messages for job: %v distributed to consumer: %v", j.jobInfo.InfoJobIdentity, j.jobInfo.Owner) }