test/simulator-group/sim-monitor.js

   1 /*
   2 #  ============LICENSE_START===============================================
   3 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   4 #  ========================================================================
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16 #  ============LICENSE_END=================================================
  17 #
  18 */
  19
  20 // Sim mon server - query the agent and the simulators for counters and other data
  21 // Presents a web page on localhost:9999/mon
  22
  23 var LOCALHOST="http://127.0.0.1:"
  24 var MRSTUB_PORT="3905"
  25 var AGENT_PORT="8081"
  26 var CR_PORT="8090"
  27 var ECS_PORT="8083"
  28 var PRODSTUB_PORT="8092"
  29 var RC_PORT="8680"
  30
  31 var http = require('http');
  32
  33 var express = require('express');
  34 const { POINT_CONVERSION_HYBRID } = require('constants')
  35 var app = express();
  36 var fieldSize=32;
  37
  38 var flagstore={}
  39
  40 //I am alive
  41 app.get("/",function(req, res){
  42         res.send("ok");
  43 })
  44
  45 //Get parameter value from other server
  46 function getSimCtr(url, index, cb) {
  47     var data = '';
  48
  49     try {
  50         http.get(url, (resp) => {
  51             // A chunk of data has been recieved.
  52             resp.on('data', (chunk) => {
  53                 data += chunk;
  54             });
  55
  56             // The whole response has been received.
  57             resp.on('end', () => {
  58                 var code=resp.statusCode
  59                 if (code > 199 && code < 300) {
  60                     cb(data, index);
  61                 } else {
  62                     cb("not found", index);
  63                 }
  64             });
  65
  66         }).on("error", (err) => {
  67             console.log("Error: " + err.message);
  68             cb("no response", index);
  69         });
  70     } catch(err) {
  71         cb("no response", index);
  72     }
  73 }
  74
  75
  76 //Format a comma separated list of data to a html-safe string with fixed fieldsizes
  77 function formatDataRow(commaList) {
  78         var str = "";
  79         var tmp=commaList.split(',');
  80     for(var i=0;i<tmp.length;i++) {
  81         var data=tmp[i];
  82         var len = fieldSize-data.length;
  83         while(len>0) {
  84             data = data+"&nbsp;";
  85             len--;
  86         }
  87         str=str+data+"&nbsp;&nbsp;&nbsp;";
  88      }
  89         return str;
  90 }
  91
  92 //Format a comma separated list of ids to a html-safe string with fixed fieldsizes
  93 function formatIdRow(commaList) {
  94         var str = "";
  95         var tmp=commaList.split(',');
  96     for(var i=0;i<tmp.length;i++) {
  97         tmp[i] = tmp[i].trim();
  98         var data="&lt"+tmp[i]+"&gt";
  99         var len = fieldSize+4-data.length;
 100         while(len>0) {
 101             data = data+"&nbsp;";
 102             len--;
 103         }
 104         str=str+data+"&nbsp;&nbsp;&nbsp;";
 105     }
 106         return str;
 107 }
 108
 109 //Format a list of ids to a html-safe string in compact format
 110 function formatIdRowCompact(commaList) {
 111     if (commaList == undefined) {
 112         commaList= "";
 113     }
 114         var str = "";
 115         var tmp=commaList.split(',');
 116     for(var i=0;i<tmp.length;i++) {
 117         tmp[i] = tmp[i].trim();
 118         var data="&lt"+tmp[i]+"&gt";
 119         str=str+data+"&nbsp;";
 120     }
 121         return str;
 122 }
 123
 124 //Pad a string upto a certain size using a pad string
 125 function padding(val, size, pad) {
 126         var s=""+val;
 127         for(var i=s.length;i<size;i++) {
 128                 s=s+pad
 129         }
 130         return s;
 131 }
 132
 133 //Function to check if the previous call has returned, if so return true, if not return false
 134 //For preventing multiple calls to slow containers.
 135 function checkFunctionFlag(flag) {
 136     if (flagstore.hasOwnProperty(flag)) {
 137         if (flagstore[flag] == 0) {
 138             flagstore[flag]=1
 139             return true
 140         } else if (flagstore[flag] > 10) {
 141             //Reset flag after ten attempts
 142             console.log("Force release flag "+flag)
 143             flagstore[flag]=1
 144             return true
 145         } else {
 146             //Previous call not returned
 147             console.log("Flag not available "+flag)
 148             flagstore[flag]=flagstore[flag]+1
 149             return false
 150         }
 151     } else {
 152         flagstore[flag]=1
 153         return true
 154     }
 155 }
 156 //Clear flag for parameter
 157 function clearFlag(flag) {
 158     flagstore[flag]=0
 159 }
 160
 161 //Status variables, for parameters values fetched from other simulators
 162 var mr1="", mr2="", mr3="", mr4="", mr5="", mr6="";
 163
 164 //Status variables for agent
 165 var ag1=""
 166 var ag2=""
 167 var ag3=""
 168 var ag4=""
 169 var ag5=""
 170
 171 //Status variables for callback receiver
 172 var cr1=""
 173 var cr2=""
 174 var cr3=""
 175
 176
 177 //Container names and ports of the ric simulator
 178 var simnames=[]
 179 var simports=[]
 180
 181 //Status variables for each ric simulator
 182 var simvar1=[]
 183 var simvar2=[]
 184 var simvar3=[]
 185 var simvar4=[]
 186 var simvar5=[]
 187 var simvar6=[]
 188
 189 //Status variables, for parameters values fetched from ecs
 190 var ecs1="", ecs2="", ecs3="", ecs4="", ecs_types="-", ecs_producers="-";
 191 var ecs_producer_arr=new Array(0)
 192 var ecs_producer_type_arr=new Array(0)
 193 var ecs_producer_jobs_arr=new Array(0)
 194 var ecs_producer_status_arr=new Array(0)
 195 var ecs_jobs=new Array(0)
 196 var ecs_job_status=new Array(0)
 197
 198 //Status variables, for parameters values fetched from prodstub
 199 var ps2="", ps3="", ps4="", ps_types="-", ps_producers="-";
 200 var ps_producer_type_arr=new Array(0)
 201 var ps_producer_jobs_arr=new Array(0)
 202 var ps_producer_delivery_arr=new Array(0)
 203
 204 //Full CR DB
 205 var cr_db={}
 206
 207 //Counts the number of get request for the html page
 208 var getCtr=0
 209
 210 var refreshCount_pol=-1
 211
 212 var refreshCount_ecs=-1
 213
 214 var refreshCount_cr=-1
 215
 216 var refreshCount_rc=-1
 217
 218 var ricbasename="ricsim"
 219
 220 var rc_services=""
 221
 222 function fetchAllMetrics_pol() {
 223
 224     console.log("Fetching policy metrics " + refreshCount_pol)
 225
 226     if (refreshCount_pol < 0) {
 227         refreshCount_pol = -1
 228         return
 229     } else {
 230         refreshCount_pol = refreshCount_pol - 1
 231     }
 232     setTimeout(() => {
 233
 234         if (getCtr%3 == 0) {
 235             //Extract the port numbers from the running simulators, for every 3 calls
 236             const { exec } = require('child_process');
 237             exec('docker ps --filter "name='+ricbasename+'" --format "{{.Names}} {{.Ports}}" | sed s/0.0.0.0:// | cut -d \'>\' -f1 | sed \'s/[[-]]*$//\'', (err, stdout, stderr) => {
 238
 239                 var simulators = ""
 240                 simulators=`${stdout}`.replace(/(\r\n|\n|\r)/gm," ");
 241                 simulators=simulators.trim();
 242                 var sims=simulators.split(" ")
 243                 simnames=[]
 244                 simports=[]
 245                 for(var i=0;i<sims.length;i=i+2) {
 246                     simnames[i/2]=sims[i]
 247                     simports[i/2]=sims[i+1]
 248                 }
 249             });
 250         }
 251         getCtr=getCtr+1
 252
 253         //Get metric values from the simulators
 254         for(var index=0;index<simnames.length;index++) {
 255
 256             if (checkFunctionFlag("simvar1_"+index)) {
 257                 getSimCtr(LOCALHOST+simports[index]+"/counter/num_instances", index, function(data, idx) {
 258                     simvar1[idx] = data;
 259                     clearFlag("simvar1_"+idx)
 260                 });
 261             }
 262             if (checkFunctionFlag("simvar2_"+index)) {
 263                 getSimCtr(LOCALHOST+simports[index]+"/counter/num_types", index, function(data,idx) {
 264                     simvar2[idx] = data;
 265                     clearFlag("simvar2_"+idx)
 266                 });
 267             }
 268             if (checkFunctionFlag("simvar3_"+index)) {
 269                 getSimCtr(LOCALHOST+simports[index]+"/policytypes", index, function(data,idx) {
 270                     data=data.replace(/\[/g,'');
 271                     data=data.replace(/\]/g,'');
 272                     data=data.replace(/ /g,'');
 273                     data=data.replace(/\"/g,'');
 274                     simvar3[idx] = data;
 275                     clearFlag("simvar3_"+idx)
 276                 });
 277             }
 278             if (checkFunctionFlag("simvar4_"+index)) {
 279                 getSimCtr(LOCALHOST+simports[index]+"/counter/interface", index, function(data,idx) {
 280                     simvar4[idx] = data;
 281                     clearFlag("simvar4_"+idx)
 282                 });
 283             }
 284             if (checkFunctionFlag("simvar5_"+index)) {
 285                 getSimCtr(LOCALHOST+simports[index]+"/counter/remote_hosts", index, function(data,idx) {
 286                     simvar5[idx] = data;
 287                     clearFlag("simvar5_"+idx)
 288                 });
 289             }
 290             if (checkFunctionFlag("simvar6_"+index)) {
 291                 getSimCtr(LOCALHOST+simports[index]+"/counter/datadelivery", index, function(data,idx) {
 292                     simvar6[idx] = data;
 293                     clearFlag("simvar6_"+idx)
 294                 });
 295             }
 296         }
 297
 298         //MR - get metrics values from the MR stub
 299         if (checkFunctionFlag("mr1")) {
 300             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/requests_submitted", 0, function(data, idx) {
 301                 mr1 = data;
 302                 clearFlag("mr1")
 303             });
 304         }
 305         if (checkFunctionFlag("mr2")) {
 306             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/requests_fetched", 0, function(data, idx) {
 307                 mr2 = data;
 308                 clearFlag("mr2")
 309             });
 310         }
 311         if (checkFunctionFlag("mr3")) {
 312             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/current_requests", 0, function(data, idx) {
 313                 mr3 = data;
 314                 clearFlag("mr3")
 315             });
 316         }
 317         if (checkFunctionFlag("mr4")) {
 318             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/responses_submitted", 0, function(data, idx) {
 319                 mr4 = data;
 320                 clearFlag("mr4")
 321             });
 322         }
 323         if (checkFunctionFlag("mr5")) {
 324             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/responses_fetched", 0, function(data, idx) {
 325                 mr5 = data;
 326                 clearFlag("mr5")
 327             });
 328         }
 329         if (checkFunctionFlag("mr6")) {
 330             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/current_responses", 0, function(data, idx) {
 331                 mr6 = data;
 332                 clearFlag("mr6")
 333             });
 334         }
 335
 336         //CR - get metrics values from the callbackreceiver
 337         if (checkFunctionFlag("cr1")) {
 338             getSimCtr(LOCALHOST+CR_PORT+"/counter/received_callbacks", 0, function(data, idx) {
 339                 cr1 = data;
 340                 clearFlag("cr1")
 341             });
 342         }
 343         if (checkFunctionFlag("cr2")) {
 344             getSimCtr(LOCALHOST+CR_PORT+"/counter/fetched_callbacks", 0, function(data, idx) {
 345                 cr2 = data;
 346                 clearFlag("cr2")
 347             });
 348         }
 349         if (checkFunctionFlag("cr3")) {
 350             getSimCtr(LOCALHOST+CR_PORT+"/counter/current_messages", 0, function(data, idx) {
 351                 cr3 = data;
 352                 clearFlag("cr3")
 353             });
 354         }
 355         //Agent - more get metrics from the agent
 356         if (checkFunctionFlag("ag1")) {
 357             getSimCtr(LOCALHOST+AGENT_PORT+"/status", 0, function(data, idx) {
 358                 ag1 = data;
 359                 clearFlag("ag1")
 360             });
 361         }
 362         if (checkFunctionFlag("ag2")) {
 363             getSimCtr(LOCALHOST+AGENT_PORT+"/services", 0, function(data, idx) {
 364                 ag2="";
 365                 try {
 366                     var jd=JSON.parse(data);
 367                     for(var key in jd) {
 368                         if (ag2.length > 1) {
 369                             ag2=ag2+", "
 370                         }
 371                         ag2=ag2+(jd[key]["serviceName"]).trim()
 372                     }
 373                 }
 374                 catch (err) {
 375                     ag2=data
 376                 }
 377                 clearFlag("ag2")
 378             });
 379         }
 380         if (checkFunctionFlag("ag3")) {
 381             getSimCtr(LOCALHOST+AGENT_PORT+"/policy_types", 0, function(data, idx) {
 382                 ag3="";
 383                 try {
 384                     var jd=JSON.parse(data);
 385                     for(var key in jd) {
 386                         if (ag3.length > 0) {
 387                             ag3=ag3+", "
 388                         }
 389                         ag3=ag3+jd[key].trim()
 390                     }
 391                 }
 392                 catch (err) {
 393                     ag3=""
 394                 }
 395                 clearFlag("ag3")
 396             });
 397         }
 398
 399         if (checkFunctionFlag("ag4")) {
 400             getSimCtr(LOCALHOST+AGENT_PORT+"/policy_ids", 0, function(data, idx) {
 401                 try {
 402                     var jd=JSON.parse(data);
 403                     ag4=""+jd.length
 404                 }
 405                 catch (err) {
 406                     ag4=""
 407                 }
 408                 clearFlag("ag4")
 409             });
 410         }
 411
 412         if (checkFunctionFlag("ag5")) {
 413             getSimCtr(LOCALHOST+AGENT_PORT+"/rics", 0, function(data, idx) {
 414                 try {
 415                     var jd=JSON.parse(data);
 416                     ag5=""+jd.length
 417                 }
 418                 catch (err) {
 419                     ag5=""
 420                 }
 421                 clearFlag("ag5")
 422             });
 423         }
 424
 425         fetchAllMetrics_pol();
 426
 427     }, 500)
 428 }
 429
 430 function fetchAllMetrics_ecs() {
 431
 432     console.log("Fetching enrichment metrics - timer:" + refreshCount_ecs)
 433
 434     if (refreshCount_ecs < 0) {
 435         refreshCount_ecs = -1
 436         return
 437     } else {
 438         refreshCount_ecs = refreshCount_ecs - 1
 439     }
 440     setTimeout(() => {
 441
 442         if (checkFunctionFlag("ecs_stat")) {
 443             getSimCtr(LOCALHOST+ECS_PORT+"/status", 0, function(data, index) {
 444                 try {
 445                     var jd=JSON.parse(data);
 446                     ecs1=jd["status"]
 447                     ecs2=""+jd["no_of_producers"]
 448                     ecs3=""+jd["no_of_types"]
 449                     ecs4=""+jd["no_of_jobs"]
 450                 }
 451                 catch (err) {
 452                     ecs1="error response"
 453                     ecs2="error response"
 454                     ecs3="error response"
 455                     ecs4="error response"
 456                 }
 457             });
 458             clearFlag("ecs_stat")
 459         }
 460         if (checkFunctionFlag("ecs_types")) {
 461             getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eitypes", 0, function(data, index) {
 462                 var tmp_ecs_types="-"
 463                 try {
 464                     var jd=JSON.parse(data);
 465                     for(var i=0;i<jd.length;i++) {
 466                         if (tmp_ecs_types.length == 1) {
 467                             tmp_ecs_types=""
 468                         }
 469                         tmp_ecs_types=""+tmp_ecs_types+jd[i]+" "
 470                     }
 471                 }
 472                 catch (err) {
 473                     tmp_ecs_types="error response"
 474                 }
 475                 ecs_types = tmp_ecs_types
 476             });
 477             clearFlag("ecs_types")
 478         }
 479         if (checkFunctionFlag("ecs_producers")) {
 480             getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers", 0, function(data, index) {
 481                 var tmp_ecs_producers="-"
 482                 try {
 483                     var jd=JSON.parse(data);
 484                     var tmp_ecs_producer_arr=new Array(jd.length)
 485                     for(var i=0;i<jd.length;i++) {
 486                         if (tmp_ecs_producers.length == 1) {
 487                             tmp_ecs_producers=""
 488                         }
 489                         tmp_ecs_producers=""+tmp_ecs_producers+jd[i]+" "
 490                         tmp_ecs_producer_arr[i]=jd[i]
 491                     }
 492                     ecs_producer_arr = tmp_ecs_producer_arr
 493                     ecs_producers = tmp_ecs_producers
 494                 }
 495                 catch (err) {
 496                     ecs_producers="error response"
 497                     ecs_producer_arr=new Array(0)
 498                 }
 499             });
 500             clearFlag("ecs_producers")
 501         }
 502         if (checkFunctionFlag("ecs_data")) {
 503             try {
 504                 var tmp_ecs_producer_type_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 505                 for(var x=0;x<tmp_ecs_producer_type_arr.length;x++) {
 506                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_type_arr[x], x, function(data, idx) {
 507                         var row=""+tmp_ecs_producer_type_arr[idx]+" : "
 508                         try {
 509                             var jd=JSON.parse(data);
 510                             var jda=jd["supported_ei_types"]
 511                             for(var j=0;j<jda.length;j++) {
 512                                 row=""+row+jda[j]["ei_type_identity"]+" "
 513                             }
 514                             tmp_ecs_producer_type_arr[idx]=row
 515                         }
 516                         catch (err) {
 517                             tmp_ecs_producer_type_arr=new Array(0)
 518                         }
 519                     });
 520                 }
 521                 ecs_producer_type_arr = tmp_ecs_producer_type_arr
 522             } catch (err) {
 523                 ecs_producer_type_arr=new Array(0)
 524             }
 525             try {
 526                 var tmp_ecs_producer_jobs_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 527                 for(x=0;x<tmp_ecs_producer_jobs_arr.length;x++) {
 528                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_jobs_arr[x]+"/eijobs", x, function(data, idx) {
 529                         var row=""+tmp_ecs_producer_jobs_arr[idx]+" : "
 530                         try {
 531                             var jd=JSON.parse(data);
 532                             for(var j=0;j<jd.length;j++) {
 533                                 var jda=jd[j]
 534                                 row=""+row+jda["ei_job_identity"]+"("+jda["ei_type_identity"]+") "
 535                             }
 536                             tmp_ecs_producer_jobs_arr[idx]=row
 537                         }
 538                         catch (err) {
 539                             tmp_ecs_producer_jobs_arr=new Array(0)
 540                         }
 541                     });
 542                 }
 543                 ecs_producer_jobs_arr = tmp_ecs_producer_jobs_arr
 544             } catch (err) {
 545                 ecs_producer_jobs_arr=new Array(0)
 546             }
 547
 548             try {
 549                 var tmp_ecs_producer_status_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 550                 for(x=0;x<tmp_ecs_producer_status_arr.length;x++) {
 551                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_status_arr[x]+"/status", x, function(data, idx) {
 552                         var row=""+tmp_ecs_producer_status_arr[idx]+" : "
 553                         try {
 554                             var jd=JSON.parse(data);
 555                             row=""+row+jd["operational_state"]
 556                             tmp_ecs_producer_status_arr[idx]=row
 557                         }
 558                         catch (err) {
 559                             tmp_ecs_producer_status_arr=new Array(0)
 560                         }
 561                     });
 562                 }
 563                 ecs_producer_status_arr = tmp_ecs_producer_status_arr
 564             } catch (err) {
 565                 ecs_producer_status_arr=new Array(0)
 566             }
 567             clearFlag("ecs_data")
 568         }
 569         if (checkFunctionFlag("ecs_jobs")) {
 570             getSimCtr(LOCALHOST+ECS_PORT+"/A1-EI/v1/eijobs", 0, function(data, index) {
 571                 try {
 572                     var jd=JSON.parse(data);
 573                     var tmpArr=new Array(jd.length)
 574                     for(var i=0;i<jd.length;i++) {
 575                         tmpArr[i]=jd[i]
 576                     }
 577                     ecs_jobs=tmpArr
 578                 }
 579                 catch (err) {
 580                     ecs_jobs=new Array(0)
 581                 }
 582             });
 583             clearFlag("ecs_jobs")
 584         }
 585         if (checkFunctionFlag("ecs_job_status")) {
 586             try {
 587                 var tmp_ecs_job_status= JSON.parse(JSON.stringify(ecs_jobs))
 588                 for(x=0;x<tmp_ecs_job_status.length;x++) {
 589                     getSimCtr(LOCALHOST+ECS_PORT+"/A1-EI/v1/eijobs/"+tmp_ecs_job_status[x]+"/status", x, function(data, idx) {
 590                         try {
 591                             var jd=JSON.parse(data);
 592                             tmp_ecs_job_status[idx]=""+tmp_ecs_job_status[idx]+":"+jd["eiJobStatus"]
 593                         }
 594                         catch (err) {
 595                             tmp_ecs_job_status="-"
 596                         }
 597                     });
 598                 }
 599                 ecs_job_status = tmp_ecs_job_status
 600             } catch (err) {
 601                 ecs_job_status="-"
 602             }
 603             clearFlag("ecs_job_status")
 604         }
 605         if (checkFunctionFlag("prodstub_stat")) {
 606             getSimCtr(LOCALHOST+PRODSTUB_PORT+"/status", x, function(data, idx) {
 607                 var ctr2_map=new Map()
 608                 var ctr3_map=new Map()
 609                 var ctr2=0
 610                 var ctr4=0
 611                 var tmp_ps_producers=""
 612                 var tmp_ps_types=""
 613                 var tmp_ps_producer_type_arr=new Array()
 614                 var tmp_ps_producer_jobs_arr=new Array()
 615                 var tmp_ps_producer_delivery_arr=new Array()
 616                 var tmp_ps2=""
 617                 var tmp_ps3=""
 618                 var tmp_ps4=""
 619                 try {
 620                     var jp=JSON.parse(data);
 621                     for(var prod_name in jp) {
 622                         ctr2_map.set(prod_name, prod_name)
 623                         ctr2 += 1
 624                         var jj=jp[prod_name]
 625                         var row=""+prod_name+" : "
 626                         var rowj=""+prod_name+" : "
 627                         var rowd=""+prod_name+" : "
 628                         tmp_ps_producers += prod_name + " "
 629                         for(var ji in jj) {
 630                             if (ji == "types") {
 631                                 var ta=jj[ji]
 632                                 for(var i=0;i<ta.length;i++) {
 633                                     ctr3_map.set(ta[i], ta[i])
 634                                     row += " "+ta[i]
 635                                 }
 636                             } else if (ji == "supervision_response") {
 637                                 //Do nothing
 638                             } else if (ji == "supervision_counter") {
 639                                 //Do nothing
 640                             } else if (ji == "types") {
 641                                 //Do nothing
 642                             } else {
 643                                 ctr4 += 1
 644                                 rowj += " "+ji
 645                                 rowd += " "+ji
 646                                 var job_data=jj[ji]["json"]
 647                                 if (job_data != undefined) {
 648                                     rowj += "("+job_data["ei_type_identity"]+")"
 649                                 }
 650                                 rowd += "("+jj[ji]["delivery_attempts"]+")"
 651                             }
 652                         }
 653                         tmp_ps_producer_type_arr[(ctr2-1)]=row
 654                         tmp_ps_producer_jobs_arr[(ctr2-1)]=rowj
 655                         tmp_ps_producer_delivery_arr[(ctr2-1)]=rowd
 656                     }
 657                     tmp_ps2=""+ctr2_map.size
 658                     tmp_ps3=""+ctr3_map.size
 659                     for(const [key, value] of ctr3_map.entries()) {
 660                         tmp_ps_types += key + " "
 661                     }
 662                     tmp_ps4=""+ctr4
 663
 664                     ps_producers=tmp_ps_producers
 665                     ps_types=tmp_ps_types
 666                     ps_producer_type_arr=tmp_ps_producer_type_arr
 667                     ps_producer_jobs_arr=tmp_ps_producer_jobs_arr
 668                     ps_producer_delivery_arr=tmp_ps_producer_delivery_arr
 669                     ps2=tmp_ps2
 670                     ps3=tmp_ps3
 671                     ps4=tmp_ps4
 672                 }
 673                 catch (err) {
 674                     ps_producers="error response"
 675                     ps_types="error response"
 676                     ps_producer_type_arr=new Array()
 677                     ps_producer_jobs_arr=new Array()
 678                     ps_producer_delivery_arr=new Array()
 679                     ps2="error response"
 680                     ps3="error response"
 681                     ps4="error response"
 682                 }
 683             });
 684             clearFlag("prodstub_stat")
 685         }
 686
 687         fetchAllMetrics_ecs();
 688
 689     }, 500)
 690 }
 691
 692 function fetchAllMetrics_cr() {
 693
 694     console.log("Fetching CR DB - timer:" + refreshCount_ecs)
 695
 696     if (refreshCount_cr < 0) {
 697         refreshCount_cr = -1
 698         return
 699     } else {
 700         refreshCount_cr = refreshCount_cr - 1
 701     }
 702     setTimeout(() => {
 703
 704         if (checkFunctionFlag("cr_stat")) {
 705             getSimCtr(LOCALHOST+CR_PORT+"/db", 0, function(data, index) {
 706                 try {
 707                     cr_db=JSON.parse(data);
 708                 }
 709                 catch (err) {
 710                     cr_db={}
 711                 }
 712             });
 713             clearFlag("cr_stat")
 714         }
 715         fetchAllMetrics_cr();
 716     }, 500)
 717 }
 718
 719 function fetchAllMetrics_rc() {
 720
 721     console.log("Fetching RC services - timer:" + refreshCount_ecs)
 722
 723     if (refreshCount_rc < 0) {
 724         refreshCount_rc = -1
 725         return
 726     } else {
 727         refreshCount_rc = refreshCount_rc - 1
 728     }
 729     setTimeout(() => {
 730
 731         if (checkFunctionFlag("rc_stat")) {
 732             getSimCtr(LOCALHOST+RC_PORT+"/services", 0, function(data, index) {
 733                 var tmp_serv=""
 734                 try {
 735                     var jd=JSON.parse(data);
 736                     for(var i=0;i<jd.length;i++) {
 737                         if (tmp_serv.length > 0) {
 738                             tmp_serv=tmp_serv+","
 739                         }
 740                         tmp_serv=tmp_serv+jd[i]["name"]
 741                     }
 742
 743                 }
 744                 catch (err) {
 745                     tmp_serv="no_response"
 746                 }
 747                 rc_services=tmp_serv
 748             });
 749             clearFlag("rc_stat")
 750         }
 751         fetchAllMetrics_rc();
 752     }, 500)
 753 }
 754
 755 // Monitor for CR db
 756 app.get("/mon3",function(req, res){
 757
 758     console.log("Creating CR DB page - timer: " + refreshCount_ecs)
 759
 760     if (refreshCount_cr < 0) {
 761         refreshCount_cr=5
 762         fetchAllMetrics_cr()
 763     }
 764     refreshCount_cr=5
 765     var json_str=JSON.stringify(cr_db, null, 1)
 766     var htmlStr = "<!DOCTYPE html>" +
 767     "<html>" +
 768     "<head>" +
 769       "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 770       "<title>CR DB dump</title>"+
 771       "</head>" +
 772       "<body style=\"white-space: pre-wrap\">" +
 773       json_str +
 774       "</body>" +
 775       "</html>";
 776     res.send(htmlStr);
 777 })
 778
 779 // Monitor for ECS
 780 app.get("/mon2",function(req, res){
 781
 782     console.log("Creating enrichment metrics - timer: " + refreshCount_ecs)
 783
 784     if (refreshCount_ecs < 0) {
 785         refreshCount_ecs=5
 786         fetchAllMetrics_ecs()
 787     }
 788     refreshCount_ecs=5
 789
 790     var summary=req.query.summary
 791
 792     if (summary == undefined) {
 793         return res.redirect('/mon2?summary=false');
 794     }
 795
 796   //Build web page
 797         var htmlStr = "<!DOCTYPE html>" +
 798           "<html>" +
 799           "<head>" +
 800             "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 801             "<title>Enrichment coordinator service and producer stub</title>"+
 802             "</head>" +
 803             "<body>" +
 804             "<font size=\"-3\" face=\"summary\">"
 805             if (summary == "false") {
 806                 htmlStr=htmlStr+"<p>Set query param '?summary' to true to only show summary statistics.</p>"
 807             } else {
 808                 htmlStr=htmlStr+"<p>Set query param '?summary' to false to only show full statistics</p>"
 809             }
 810             if (ecs_job_status.length > 10) {
 811                 htmlStr=htmlStr+"<div style=\"color:red\"> Avoid running the server for large number of producers and/or jobs</div>"
 812             }
 813             htmlStr=htmlStr+"</font>" +
 814             "<h3>Enrichment Coordinator Service</h3>" +
 815             "<font face=\"monospace\">" +
 816             "Status:..........." + formatDataRow(ecs1) + "<br>" +
 817             "Producers:........" + formatDataRow(ecs2) + "<br>" +
 818             "Types:............" + formatDataRow(ecs3) + "<br>" +
 819             "Jobs:............." + formatDataRow(ecs4) + "<br>" +
 820             "</font>"
 821             if (summary == "false") {
 822                 htmlStr=htmlStr+
 823                 "<h4>Details</h4>" +
 824                 "<font face=\"monospace\">" +
 825                 "Producer ids:....." + formatDataRow(ecs_producers) + "<br>" +
 826                 "Type ids:........." + formatDataRow(ecs_types) + "<br>" +
 827                 "<br>";
 828                 for(var i=0;i<ecs_producer_type_arr.length;i++) {
 829                     var tmp=ecs_producer_type_arr[i]
 830                     if (tmp != undefined) {
 831                         var s = "Producer types...." + formatDataRow(ecs_producer_type_arr[i]) + "<br>"
 832                         htmlStr=htmlStr+s
 833                     }
 834                 }
 835                 htmlStr=htmlStr+"<br>";
 836                 for(i=0;i<ecs_producer_jobs_arr.length;i++) {
 837                     tmp=ecs_producer_jobs_arr[i]
 838                     if (tmp != undefined) {
 839                         s = "Producer jobs....." + formatDataRow(ecs_producer_jobs_arr[i]) + "<br>"
 840                         htmlStr=htmlStr+s
 841                     }
 842                 }
 843                 htmlStr=htmlStr+"<br>";
 844                 for(i=0;i<ecs_producer_status_arr.length;i++) {
 845                     tmp=ecs_producer_status_arr[i]
 846                     if (tmp != undefined) {
 847                         s = "Producer status..." + formatDataRow(tmp) + "<br>"
 848                         htmlStr=htmlStr+s
 849                     }
 850                 }
 851                 htmlStr=htmlStr+"<br>";
 852                 for(i=0;i<ecs_job_status.length;i++) {
 853                     tmp=ecs_job_status[i]
 854                     if (tmp != undefined) {
 855                         s = padding("Job", 18, ".") + formatDataRow(tmp) + "<br>"
 856                         htmlStr=htmlStr+s
 857                     }
 858                 }
 859                 htmlStr=htmlStr+"<br>"+"<br>" +
 860                 "</font>"
 861             }
 862
 863             htmlStr=htmlStr+
 864             "<h3>Producer stub</h3>" +
 865             "<font face=\"monospace\">" +
 866             "Producers:........" + formatDataRow(ps2) + "<br>" +
 867             "Types:............" + formatDataRow(ps3) + "<br>" +
 868             "Jobs:............." + formatDataRow(ps4) + "<br>" +
 869             "</font>"
 870             if (summary == "false") {
 871                 htmlStr=htmlStr+
 872                 "<h4>Details</h4>" +
 873                 "<font face=\"monospace\">" +
 874                 "Producer ids:....." + formatDataRow(ps_producers) + "<br>" +
 875                 "Type ids:........." + formatDataRow(ps_types) + "<br>" +
 876                 "<br>";
 877                 for(i=0;i<ps_producer_type_arr.length;i++) {
 878                     tmp=ps_producer_type_arr[i]
 879                     if (tmp != undefined) {
 880                         s = "Producer types...." + formatDataRow(ps_producer_type_arr[i]) + "<br>"
 881                         htmlStr=htmlStr+s
 882                     }
 883                 }
 884                 htmlStr=htmlStr+"<br>";
 885                 for(i=0;i<ps_producer_jobs_arr.length;i++) {
 886                     tmp=ps_producer_jobs_arr[i]
 887                     if (tmp != undefined) {
 888                         s = "Producer jobs....." + formatDataRow(ps_producer_jobs_arr[i]) + "<br>"
 889                         htmlStr=htmlStr+s
 890                     }
 891                 }
 892                 htmlStr=htmlStr+"<br>";
 893                 for(i=0;i<ps_producer_delivery_arr.length;i++) {
 894                     tmp=ps_producer_delivery_arr[i]
 895                     if (tmp != undefined) {
 896                         s = "Producer delivery." + formatDataRow(ps_producer_delivery_arr[i]) + "<br>"
 897                         htmlStr=htmlStr+s
 898                     }
 899                 }
 900             }
 901             htmlStr=htmlStr+
 902             "</font>" +
 903            "</body>" +
 904           "</html>";
 905         res.send(htmlStr);
 906 })
 907
 908 // Monitor for policy management
 909 app.get("/mon",function(req, res){
 910
 911     console.log("Creating policy metrics page " + refreshCount_pol)
 912
 913     if (refreshCount_pol < 0) {
 914         refreshCount_pol=5
 915         fetchAllMetrics_pol()
 916     }
 917     refreshCount_pol=5
 918
 919     if (refreshCount_rc < 0) {
 920         refreshCount_rc=5
 921         fetchAllMetrics_rc()
 922     }
 923     refreshCount_rc=5
 924
 925     var bn=req.query.basename
 926
 927     if (bn == undefined) {
 928         getCtr=0
 929         return res.redirect('/mon?basename=ricsim');
 930     } else {
 931         ricbasename=bn
 932     }
 933
 934     //Build web page
 935         var htmlStr = "<!DOCTYPE html>" +
 936           "<html>" +
 937           "<head>" +
 938             "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 939             "<title>Policy Agent and simulator monitor</title>"+
 940             "</head>" +
 941             "<body>" +
 942             "<font size=\"-3\" face=\"monospace\">" +
 943             "<p>Change basename in url if other ric sim prefix is used</p>" +
 944             "</font>" +
 945             "<h3>Policy agent</h3>" +
 946             "<font face=\"monospace\">" +
 947             "Status:..............................." + formatDataRow(ag1) + "<br>" +
 948             "Services:............................." + formatIdRowCompact(ag2) + "<br>" +
 949             "Types:................................" + formatIdRowCompact(ag3) + "<br>" +
 950             "Number of instances:.................." + formatDataRow(ag4) + "<br>" +
 951             "Near-RT RICs:........................." + formatDataRow(ag5) + "<br>" +
 952             "</font>" +
 953             "<h3>MR Stub interface</h3>" +
 954             "<font face=\"monospace\">"+
 955             "Submitted requests:............................" + formatDataRow(mr1) + "<br>" +
 956             "Fetched requests:.............................." + formatDataRow(mr2) + "<br>" +
 957             "Current requests waiting:......................" + formatDataRow(mr3) + "<br>" +
 958             "Submitted responses:..........................." + formatDataRow(mr4) + "<br>" +
 959             "Fetched responses.............................." + formatDataRow(mr5) + "<br>" +
 960             "Current responses waiting......................" + formatDataRow(mr6) + "<br>" +
 961             "</font>"+
 962             "<h3>Callback receiver</h3>" +
 963             "<font face=\"monospace\">" +
 964             "Callbacks received:..................." + formatDataRow(cr1) + "<br>" +
 965             "Callbacks fetched:...................." + formatDataRow(cr2) + "<br>" +
 966             "Number of waiting callback messages:.." + formatDataRow(cr3) + "<br>" +
 967             "</font>" +
 968             "<h3>R-APP Catalogue</h3>" +
 969             "<font face=\"monospace\">" +
 970             "Services:............................." + formatIdRowCompact(rc_services) + "<br>" +
 971             "</font>" +
 972             "<h3>Near-RT RIC Simulators</h3>" +
 973             "<font face=\"monospace\">"
 974
 975             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
 976             htmlStr=htmlStr+padding("Types", 10,"&nbsp;")
 977             htmlStr=htmlStr+padding("Instances", 12,"&nbsp;")
 978             htmlStr=htmlStr+padding("Data delivery", 12,"&nbsp;")+"<br>"
 979             htmlStr=htmlStr+padding("",70,"=")+"<br>"
 980             for(var simIndex=0;simIndex<simnames.length;simIndex++) {
 981                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
 982                 htmlStr=htmlStr+padding(simvar2[simIndex],10,"&nbsp;")
 983                 htmlStr=htmlStr+padding(simvar1[simIndex],12    ,"&nbsp;")
 984                 htmlStr=htmlStr+padding(simvar6[simIndex],12,"&nbsp;")
 985                 htmlStr=htmlStr+"<br>";
 986             }
 987
 988             htmlStr=htmlStr+"<br>";
 989             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
 990             htmlStr=htmlStr+padding("Version", 20,"&nbsp;")
 991             htmlStr=htmlStr+padding("Type-IDs", 10,"&nbsp;")+"<br>"
 992             htmlStr=htmlStr+padding("",65,"=")+"<br>"
 993             for(simIndex=0;simIndex<simnames.length;simIndex++) {
 994                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
 995                 htmlStr=htmlStr+padding(simvar4[simIndex],20,"&nbsp;")
 996                 htmlStr=htmlStr+padding(formatIdRowCompact(simvar3[simIndex]),10,"&nbsp;")
 997                 htmlStr=htmlStr+"<br>";
 998             }
 999
1000             htmlStr=htmlStr+"<br>";
1001             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
1002             htmlStr=htmlStr+padding("Remote hosts", 50,"&nbsp;")+"<br>"
1003             htmlStr=htmlStr+padding("",90,"=")+"<br>"
1004             for(simIndex=0;simIndex<simnames.length;simIndex++) {
1005                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
1006                 htmlStr=htmlStr+padding(simvar5[simIndex],50,"&nbsp;")
1007                 htmlStr=htmlStr+"<br>";
1008             }
1009
1010             htmlStr=htmlStr+
1011            "</body>" +
1012           "</html>";
1013         res.send(htmlStr);
1014 })
1015
1016 var httpServer = http.createServer(app);
1017 var httpPort=9999;
1018 httpServer.listen(httpPort);
1019 console.log("Simulator monitor listening (http) at "+httpPort);
1020 console.log("Open the web page on localhost:9999/mon to view the policy statistics page.")
1021 console.log("Open the web page on localhost:9999/mon2 to view the enrichment statistics page.")
1022 console.log("Open the web page on localhost:9999/mon3 to view CR DB in json.")