test/simulator-group/sim-monitor.js

   1 /*
   2 #  ============LICENSE_START===============================================
   3 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   4 #  ========================================================================
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16 #  ============LICENSE_END=================================================
  17 #
  18 */
  19
  20 // Sim mon server - query the agent and the simulators for counters and other data
  21 // Presents a web page on localhost:9999/mon
  22
  23 var LOCALHOST="http://127.0.0.1:"
  24 var MRSTUB_PORT="3905"
  25 var AGENT_PORT="8081"
  26 var CR_PORT="8090"
  27 var ECS_PORT="8083"
  28 var PRODSTUB_PORT="8092"
  29 var RC_PORT="8680"
  30
  31 var http = require('http');
  32
  33 var express = require('express');
  34 const { POINT_CONVERSION_HYBRID } = require('constants')
  35 var app = express();
  36 var fieldSize=32;
  37
  38 var flagstore={}
  39
  40 //I am alive
  41 app.get("/",function(req, res){
  42         res.send("ok");
  43 })
  44
  45 //Get parameter value from other server
  46 function getSimCtr(url, index, cb) {
  47     var data = '';
  48
  49     try {
  50         http.get(url, (resp) => {
  51             // A chunk of data has been recieved.
  52             resp.on('data', (chunk) => {
  53                 data += chunk;
  54             });
  55
  56             // The whole response has been received.
  57             resp.on('end', () => {
  58                 var code=resp.statusCode
  59                 if (code > 199 && code < 300) {
  60                     cb(data, index);
  61                 } else {
  62                     cb("not found", index);
  63                 }
  64             });
  65
  66         }).on("error", (err) => {
  67             console.log("Error: " + err.message);
  68             cb("no response", index);
  69         });
  70     } catch(err) {
  71         cb("no response", index);
  72     }
  73 }
  74
  75
  76 //Format a comma separated list of data to a html-safe string with fixed fieldsizes
  77 function formatDataRow(commaList) {
  78         var str = "";
  79         var tmp=commaList.split(',');
  80     for(var i=0;i<tmp.length;i++) {
  81         var data=tmp[i];
  82         var len = fieldSize-data.length;
  83         while(len>0) {
  84             data = data+"&nbsp;";
  85             len--;
  86         }
  87         str=str+data+"&nbsp;&nbsp;&nbsp;";
  88      }
  89         return str;
  90 }
  91
  92 //Format a comma separated list of ids to a html-safe string with fixed fieldsizes
  93 function formatIdRow(commaList) {
  94         var str = "";
  95         var tmp=commaList.split(',');
  96     for(var i=0;i<tmp.length;i++) {
  97         tmp[i] = tmp[i].trim();
  98         var data="&lt"+tmp[i]+"&gt";
  99         var len = fieldSize+4-data.length;
 100         while(len>0) {
 101             data = data+"&nbsp;";
 102             len--;
 103         }
 104         str=str+data+"&nbsp;&nbsp;&nbsp;";
 105     }
 106         return str;
 107 }
 108
 109 //Format a list of ids to a html-safe string in compact format
 110 function formatIdRowCompact(commaList) {
 111     if (commaList == undefined) {
 112         commaList= "";
 113     }
 114         var str = "";
 115         var tmp=commaList.split(',');
 116     for(var i=0;i<tmp.length;i++) {
 117         tmp[i] = tmp[i].trim();
 118         var data="&lt"+tmp[i]+"&gt";
 119         str=str+data+"&nbsp;";
 120     }
 121         return str;
 122 }
 123
 124 //Pad a string upto a certain size using a pad string
 125 function padding(val, size, pad) {
 126         var s=""+val;
 127         for(var i=s.length;i<size;i++) {
 128                 s=s+pad
 129         }
 130         return s;
 131 }
 132
 133 //Function to check if the previous call has returned, if so return true, if not return false
 134 //For preventing multiple calls to slow containers.
 135 function checkFunctionFlag(flag) {
 136     if (flagstore.hasOwnProperty(flag)) {
 137         if (flagstore[flag] == 0) {
 138             flagstore[flag]=1
 139             return true
 140         } else if (flagstore[flag] > 10) {
 141             //Reset flag after ten attempts
 142             console.log("Force release flag "+flag)
 143             flagstore[flag]=1
 144             return true
 145         } else {
 146             //Previous call not returned
 147             console.log("Flag not available "+flag)
 148             flagstore[flag]=flagstore[flag]+1
 149             return false
 150         }
 151     } else {
 152         flagstore[flag]=1
 153         return true
 154     }
 155 }
 156 //Clear flag for parameter
 157 function clearFlag(flag) {
 158     flagstore[flag]=0
 159 }
 160
 161 //Status variables, for parameters values fetched from other simulators
 162 var mr1="", mr2="", mr3="", mr4="", mr5="", mr6="";
 163
 164 //Status variables for agent
 165 var ag1=""
 166 var ag2=""
 167 var ag3=""
 168 var ag4=""
 169 var ag5=""
 170
 171 //Status variables for callback receiver
 172 var cr1=""
 173 var cr2=""
 174 var cr3=""
 175
 176
 177 //Container names and ports of the ric simulator
 178 var simnames=[]
 179 var simports=[]
 180
 181 //Status variables for each ric simulator
 182 var simvar1=[]
 183 var simvar2=[]
 184 var simvar3=[]
 185 var simvar4=[]
 186 var simvar5=[]
 187 var simvar6=[]
 188
 189 //Status variables, for parameters values fetched from ecs
 190 var ecs1="", ecs2="", ecs3="", ecs4="", ecs_types="-", ecs_producers="-";
 191 var ecs_producer_arr=new Array(0)
 192 var ecs_producer_type_arr=new Array(0)
 193 var ecs_producer_jobs_arr=new Array(0)
 194 var ecs_producer_status_arr=new Array(0)
 195 var ecs_jobs=new Array(0)
 196 var ecs_job_status=new Array(0)
 197
 198 //Status variables, for parameters values fetched from prodstub
 199 var ps2="", ps3="", ps4="", ps_types="-", ps_producers="-";
 200 var ps_producer_type_arr=new Array(0)
 201 var ps_producer_jobs_arr=new Array(0)
 202 var ps_producer_delivery_arr=new Array(0)
 203
 204 //Full CR DB
 205 var cr_db={}
 206
 207 //Counts the number of get request for the html page
 208 var getCtr=0
 209
 210 var refreshCount_pol=-1
 211
 212 var refreshCount_ecs=-1
 213
 214 var refreshCount_cr=-1
 215
 216 var refreshCount_rc=-1
 217
 218 var ricbasename="ricsim"
 219
 220 var rc_services=""
 221
 222 var pmsprefix=""
 223
 224 function fetchAllMetrics_pol() {
 225
 226     console.log("Fetching policy metrics " + refreshCount_pol)
 227
 228     if (refreshCount_pol < 0) {
 229         refreshCount_pol = -1
 230         return
 231     } else {
 232         refreshCount_pol = refreshCount_pol - 1
 233     }
 234     setTimeout(() => {
 235
 236         if (getCtr%3 == 0) {
 237             //Extract the port numbers from the running simulators, for every 3 calls
 238             const { exec } = require('child_process');
 239             exec('docker ps --filter "name='+ricbasename+'" --filter "network=nonrtric-docker-net" --format "{{.Names}} {{.Ports}}" | sed s/0.0.0.0:// | cut -d \'>\' -f1 | sed \'s/[[-]]*$//\'', (err, stdout, stderr) => {
 240
 241                 var simulators = ""
 242                 simulators=`${stdout}`.replace(/(\r\n|\n|\r)/gm," ");
 243                 simulators=simulators.trim();
 244                 var sims=simulators.split(" ")
 245                 simnames=[]
 246                 simports=[]
 247                 for(var i=0;i<sims.length;i=i+2) {
 248                     simnames[i/2]=sims[i]
 249                     simports[i/2]=sims[i+1]
 250                 }
 251             });
 252         }
 253         getCtr=getCtr+1
 254
 255         //Get metric values from the simulators
 256         for(var index=0;index<simnames.length;index++) {
 257
 258             if (checkFunctionFlag("simvar1_"+index)) {
 259                 getSimCtr(LOCALHOST+simports[index]+"/counter/num_instances", index, function(data, idx) {
 260                     simvar1[idx] = data;
 261                     clearFlag("simvar1_"+idx)
 262                 });
 263             }
 264             if (checkFunctionFlag("simvar2_"+index)) {
 265                 getSimCtr(LOCALHOST+simports[index]+"/counter/num_types", index, function(data,idx) {
 266                     simvar2[idx] = data;
 267                     clearFlag("simvar2_"+idx)
 268                 });
 269             }
 270             if (checkFunctionFlag("simvar3_"+index)) {
 271                 getSimCtr(LOCALHOST+simports[index]+"/policytypes", index, function(data,idx) {
 272                     data=data.replace(/\[/g,'');
 273                     data=data.replace(/\]/g,'');
 274                     data=data.replace(/ /g,'');
 275                     data=data.replace(/\"/g,'');
 276                     simvar3[idx] = data;
 277                     clearFlag("simvar3_"+idx)
 278                 });
 279             }
 280             if (checkFunctionFlag("simvar4_"+index)) {
 281                 getSimCtr(LOCALHOST+simports[index]+"/counter/interface", index, function(data,idx) {
 282                     simvar4[idx] = data;
 283                     clearFlag("simvar4_"+idx)
 284                 });
 285             }
 286             if (checkFunctionFlag("simvar5_"+index)) {
 287                 getSimCtr(LOCALHOST+simports[index]+"/counter/remote_hosts", index, function(data,idx) {
 288                     simvar5[idx] = data;
 289                     clearFlag("simvar5_"+idx)
 290                 });
 291             }
 292             if (checkFunctionFlag("simvar6_"+index)) {
 293                 getSimCtr(LOCALHOST+simports[index]+"/counter/datadelivery", index, function(data,idx) {
 294                     simvar6[idx] = data;
 295                     clearFlag("simvar6_"+idx)
 296                 });
 297             }
 298         }
 299
 300         //MR - get metrics values from the MR stub
 301         if (checkFunctionFlag("mr1")) {
 302             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/requests_submitted", 0, function(data, idx) {
 303                 mr1 = data;
 304                 clearFlag("mr1")
 305             });
 306         }
 307         if (checkFunctionFlag("mr2")) {
 308             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/requests_fetched", 0, function(data, idx) {
 309                 mr2 = data;
 310                 clearFlag("mr2")
 311             });
 312         }
 313         if (checkFunctionFlag("mr3")) {
 314             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/current_requests", 0, function(data, idx) {
 315                 mr3 = data;
 316                 clearFlag("mr3")
 317             });
 318         }
 319         if (checkFunctionFlag("mr4")) {
 320             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/responses_submitted", 0, function(data, idx) {
 321                 mr4 = data;
 322                 clearFlag("mr4")
 323             });
 324         }
 325         if (checkFunctionFlag("mr5")) {
 326             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/responses_fetched", 0, function(data, idx) {
 327                 mr5 = data;
 328                 clearFlag("mr5")
 329             });
 330         }
 331         if (checkFunctionFlag("mr6")) {
 332             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/current_responses", 0, function(data, idx) {
 333                 mr6 = data;
 334                 clearFlag("mr6")
 335             });
 336         }
 337
 338         //CR - get metrics values from the callbackreceiver
 339         if (checkFunctionFlag("cr1")) {
 340             getSimCtr(LOCALHOST+CR_PORT+"/counter/received_callbacks", 0, function(data, idx) {
 341                 cr1 = data;
 342                 clearFlag("cr1")
 343             });
 344         }
 345         if (checkFunctionFlag("cr2")) {
 346             getSimCtr(LOCALHOST+CR_PORT+"/counter/fetched_callbacks", 0, function(data, idx) {
 347                 cr2 = data;
 348                 clearFlag("cr2")
 349             });
 350         }
 351         if (checkFunctionFlag("cr3")) {
 352             getSimCtr(LOCALHOST+CR_PORT+"/counter/current_messages", 0, function(data, idx) {
 353                 cr3 = data;
 354                 clearFlag("cr3")
 355             });
 356         }
 357         //Agent - more get metrics from the agent
 358         if (checkFunctionFlag("ag1")) {
 359             getSimCtr(LOCALHOST+AGENT_PORT+"/status", 0, function(data, idx) {
 360                 ag1 = data;
 361                 clearFlag("ag1")
 362             });
 363         }
 364         if (checkFunctionFlag("ag2")) {
 365             getSimCtr(LOCALHOST+AGENT_PORT+"/services", 0, function(data, idx) {
 366                 ag2="";
 367                 try {
 368                     var jd=JSON.parse(data);
 369                     for(var key in jd) {
 370                         if (ag2.length > 1) {
 371                             ag2=ag2+", "
 372                         }
 373                         ag2=ag2+(jd[key]["serviceName"]).trim()
 374                     }
 375                 }
 376                 catch (err) {
 377                     ag2=data
 378                 }
 379                 clearFlag("ag2")
 380             });
 381         }
 382         if (checkFunctionFlag("ag3")) {
 383             getSimCtr(LOCALHOST+AGENT_PORT+"/policy_types", 0, function(data, idx) {
 384                 ag3="";
 385                 try {
 386                     var jd=JSON.parse(data);
 387                     for(var key in jd) {
 388                         if (ag3.length > 0) {
 389                             ag3=ag3+", "
 390                         }
 391                         ag3=ag3+jd[key].trim()
 392                     }
 393                 }
 394                 catch (err) {
 395                     ag3=""
 396                 }
 397                 clearFlag("ag3")
 398             });
 399         }
 400
 401         if (checkFunctionFlag("ag4")) {
 402             getSimCtr(LOCALHOST+AGENT_PORT+"/policy_ids", 0, function(data, idx) {
 403                 try {
 404                     var jd=JSON.parse(data);
 405                     ag4=""+jd.length
 406                 }
 407                 catch (err) {
 408                     ag4=""
 409                 }
 410                 clearFlag("ag4")
 411             });
 412         }
 413
 414         if (checkFunctionFlag("ag5")) {
 415             getSimCtr(LOCALHOST+AGENT_PORT+"/rics", 0, function(data, idx) {
 416                 try {
 417                     var jd=JSON.parse(data);
 418                     ag5=""+jd.length
 419                 }
 420                 catch (err) {
 421                     ag5=""
 422                 }
 423                 clearFlag("ag5")
 424             });
 425         }
 426
 427         fetchAllMetrics_pol();
 428
 429     }, 500)
 430 }
 431
 432 function fetchAllMetrics_ecs() {
 433
 434     console.log("Fetching enrichment metrics - timer:" + refreshCount_ecs)
 435
 436     if (refreshCount_ecs < 0) {
 437         refreshCount_ecs = -1
 438         return
 439     } else {
 440         refreshCount_ecs = refreshCount_ecs - 1
 441     }
 442     setTimeout(() => {
 443
 444         if (checkFunctionFlag("ecs_stat")) {
 445             getSimCtr(LOCALHOST+ECS_PORT+"/status", 0, function(data, index) {
 446                 try {
 447                     var jd=JSON.parse(data);
 448                     ecs1=jd["status"]
 449                     ecs2=""+jd["no_of_producers"]
 450                     ecs3=""+jd["no_of_types"]
 451                     ecs4=""+jd["no_of_jobs"]
 452                 }
 453                 catch (err) {
 454                     ecs1="error response"
 455                     ecs2="error response"
 456                     ecs3="error response"
 457                     ecs4="error response"
 458                 }
 459             });
 460             clearFlag("ecs_stat")
 461         }
 462         if (checkFunctionFlag("ecs_types")) {
 463             getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eitypes", 0, function(data, index) {
 464                 var tmp_ecs_types="-"
 465                 try {
 466                     var jd=JSON.parse(data);
 467                     for(var i=0;i<jd.length;i++) {
 468                         if (tmp_ecs_types.length == 1) {
 469                             tmp_ecs_types=""
 470                         }
 471                         tmp_ecs_types=""+tmp_ecs_types+jd[i]+" "
 472                     }
 473                 }
 474                 catch (err) {
 475                     tmp_ecs_types="error response"
 476                 }
 477                 ecs_types = tmp_ecs_types
 478             });
 479             clearFlag("ecs_types")
 480         }
 481         if (checkFunctionFlag("ecs_producers")) {
 482             getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers", 0, function(data, index) {
 483                 var tmp_ecs_producers="-"
 484                 try {
 485                     var jd=JSON.parse(data);
 486                     var tmp_ecs_producer_arr=new Array(jd.length)
 487                     for(var i=0;i<jd.length;i++) {
 488                         if (tmp_ecs_producers.length == 1) {
 489                             tmp_ecs_producers=""
 490                         }
 491                         tmp_ecs_producers=""+tmp_ecs_producers+jd[i]+" "
 492                         tmp_ecs_producer_arr[i]=jd[i]
 493                     }
 494                     ecs_producer_arr = tmp_ecs_producer_arr
 495                     ecs_producers = tmp_ecs_producers
 496                 }
 497                 catch (err) {
 498                     ecs_producers="error response"
 499                     ecs_producer_arr=new Array(0)
 500                 }
 501             });
 502             clearFlag("ecs_producers")
 503         }
 504         if (checkFunctionFlag("ecs_data")) {
 505             try {
 506                 var tmp_ecs_producer_type_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 507                 for(var x=0;x<tmp_ecs_producer_type_arr.length;x++) {
 508                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_type_arr[x], x, function(data, idx) {
 509                         var row=""+tmp_ecs_producer_type_arr[idx]+" : "
 510                         try {
 511                             var jd=JSON.parse(data);
 512                             var jda=jd["supported_ei_types"]
 513                             for(var j=0;j<jda.length;j++) {
 514                                 row=""+row+jda[j]+" "
 515
 516                             }
 517                             tmp_ecs_producer_type_arr[idx]=row
 518                         }
 519                         catch (err) {
 520                             tmp_ecs_producer_type_arr=new Array(0)
 521                         }
 522                     });
 523                 }
 524                 ecs_producer_type_arr = tmp_ecs_producer_type_arr
 525             } catch (err) {
 526                 ecs_producer_type_arr=new Array(0)
 527             }
 528             try {
 529                 var tmp_ecs_producer_jobs_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 530                 for(x=0;x<tmp_ecs_producer_jobs_arr.length;x++) {
 531                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_jobs_arr[x]+"/eijobs", x, function(data, idx) {
 532                         var row=""+tmp_ecs_producer_jobs_arr[idx]+" : "
 533                         try {
 534                             var jd=JSON.parse(data);
 535                             for(var j=0;j<jd.length;j++) {
 536                                 var jda=jd[j]
 537                                 row=""+row+jda["ei_job_identity"]+"("+jda["ei_type_identity"]+") "
 538                             }
 539                             tmp_ecs_producer_jobs_arr[idx]=row
 540                         }
 541                         catch (err) {
 542                             tmp_ecs_producer_jobs_arr=new Array(0)
 543                         }
 544                     });
 545                 }
 546                 ecs_producer_jobs_arr = tmp_ecs_producer_jobs_arr
 547             } catch (err) {
 548                 ecs_producer_jobs_arr=new Array(0)
 549             }
 550
 551             try {
 552                 var tmp_ecs_producer_status_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 553                 for(x=0;x<tmp_ecs_producer_status_arr.length;x++) {
 554                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_status_arr[x]+"/status", x, function(data, idx) {
 555                         var row=""+tmp_ecs_producer_status_arr[idx]+" : "
 556                         try {
 557                             var jd=JSON.parse(data);
 558                             row=""+row+jd["operational_state"]
 559                             tmp_ecs_producer_status_arr[idx]=row
 560                         }
 561                         catch (err) {
 562                             tmp_ecs_producer_status_arr=new Array(0)
 563                         }
 564                     });
 565                 }
 566                 ecs_producer_status_arr = tmp_ecs_producer_status_arr
 567             } catch (err) {
 568                 ecs_producer_status_arr=new Array(0)
 569             }
 570             clearFlag("ecs_data")
 571         }
 572         if (checkFunctionFlag("ecs_jobs")) {
 573             getSimCtr(LOCALHOST+ECS_PORT+"/A1-EI/v1/eijobs", 0, function(data, index) {
 574                 try {
 575                     var jd=JSON.parse(data);
 576                     var tmpArr=new Array(jd.length)
 577                     for(var i=0;i<jd.length;i++) {
 578                         tmpArr[i]=jd[i]
 579                     }
 580                     ecs_jobs=tmpArr
 581                 }
 582                 catch (err) {
 583                     ecs_jobs=new Array(0)
 584                 }
 585             });
 586             clearFlag("ecs_jobs")
 587         }
 588         if (checkFunctionFlag("ecs_job_status")) {
 589             try {
 590                 var tmp_ecs_job_status= JSON.parse(JSON.stringify(ecs_jobs))
 591                 for(x=0;x<tmp_ecs_job_status.length;x++) {
 592                     getSimCtr(LOCALHOST+ECS_PORT+"/A1-EI/v1/eijobs/"+tmp_ecs_job_status[x]+"/status", x, function(data, idx) {
 593                         try {
 594                             var jd=JSON.parse(data);
 595                             tmp_ecs_job_status[idx]=""+tmp_ecs_job_status[idx]+":"+jd["eiJobStatus"]
 596                         }
 597                         catch (err) {
 598                             tmp_ecs_job_status="-"
 599                         }
 600                     });
 601                 }
 602                 ecs_job_status = tmp_ecs_job_status
 603             } catch (err) {
 604                 ecs_job_status="-"
 605             }
 606             clearFlag("ecs_job_status")
 607         }
 608         if (checkFunctionFlag("prodstub_stat")) {
 609             getSimCtr(LOCALHOST+PRODSTUB_PORT+"/status", x, function(data, idx) {
 610                 var ctr2_map=new Map()
 611                 var ctr3_map=new Map()
 612                 var ctr2=0
 613                 var ctr4=0
 614                 var tmp_ps_producers=""
 615                 var tmp_ps_types=""
 616                 var tmp_ps_producer_type_arr=new Array()
 617                 var tmp_ps_producer_jobs_arr=new Array()
 618                 var tmp_ps_producer_delivery_arr=new Array()
 619                 var tmp_ps2=""
 620                 var tmp_ps3=""
 621                 var tmp_ps4=""
 622                 try {
 623                     var jp=JSON.parse(data);
 624                     for(var prod_name in jp) {
 625                         ctr2_map.set(prod_name, prod_name)
 626                         ctr2 += 1
 627                         var jj=jp[prod_name]
 628                         var row=""+prod_name+" : "
 629                         var rowj=""+prod_name+" : "
 630                         var rowd=""+prod_name+" : "
 631                         tmp_ps_producers += prod_name + " "
 632                         for(var ji in jj) {
 633                             if (ji == "types") {
 634                                 var ta=jj[ji]
 635                                 for(var i=0;i<ta.length;i++) {
 636                                     ctr3_map.set(ta[i], ta[i])
 637                                     row += " "+ta[i]
 638                                 }
 639                             } else if (ji == "supervision_response") {
 640                                 //Do nothing
 641                             } else if (ji == "supervision_counter") {
 642                                 //Do nothing
 643                             } else if (ji == "types") {
 644                                 //Do nothing
 645                             } else {
 646                                 ctr4 += 1
 647                                 rowj += " "+ji
 648                                 rowd += " "+ji
 649                                 var job_data=jj[ji]["json"]
 650                                 if (job_data != undefined) {
 651                                     rowj += "("+job_data["ei_type_identity"]+")"
 652                                 }
 653                                 rowd += "("+jj[ji]["delivery_attempts"]+")"
 654                             }
 655                         }
 656                         tmp_ps_producer_type_arr[(ctr2-1)]=row
 657                         tmp_ps_producer_jobs_arr[(ctr2-1)]=rowj
 658                         tmp_ps_producer_delivery_arr[(ctr2-1)]=rowd
 659                     }
 660                     tmp_ps2=""+ctr2_map.size
 661                     tmp_ps3=""+ctr3_map.size
 662                     for(const [key, value] of ctr3_map.entries()) {
 663                         tmp_ps_types += key + " "
 664                     }
 665                     tmp_ps4=""+ctr4
 666
 667                     ps_producers=tmp_ps_producers
 668                     ps_types=tmp_ps_types
 669                     ps_producer_type_arr=tmp_ps_producer_type_arr
 670                     ps_producer_jobs_arr=tmp_ps_producer_jobs_arr
 671                     ps_producer_delivery_arr=tmp_ps_producer_delivery_arr
 672                     ps2=tmp_ps2
 673                     ps3=tmp_ps3
 674                     ps4=tmp_ps4
 675                 }
 676                 catch (err) {
 677                     ps_producers="error response"
 678                     ps_types="error response"
 679                     ps_producer_type_arr=new Array()
 680                     ps_producer_jobs_arr=new Array()
 681                     ps_producer_delivery_arr=new Array()
 682                     ps2="error response"
 683                     ps3="error response"
 684                     ps4="error response"
 685                 }
 686             });
 687             clearFlag("prodstub_stat")
 688         }
 689
 690         fetchAllMetrics_ecs();
 691
 692     }, 500)
 693 }
 694
 695 function fetchAllMetrics_cr() {
 696
 697     console.log("Fetching CR DB - timer:" + refreshCount_ecs)
 698
 699     if (refreshCount_cr < 0) {
 700         refreshCount_cr = -1
 701         return
 702     } else {
 703         refreshCount_cr = refreshCount_cr - 1
 704     }
 705     setTimeout(() => {
 706
 707         if (checkFunctionFlag("cr_stat")) {
 708             getSimCtr(LOCALHOST+CR_PORT+"/db", 0, function(data, index) {
 709                 try {
 710                     cr_db=JSON.parse(data);
 711                 }
 712                 catch (err) {
 713                     cr_db={}
 714                 }
 715             });
 716             clearFlag("cr_stat")
 717         }
 718         fetchAllMetrics_cr();
 719     }, 500)
 720 }
 721
 722 function fetchAllMetrics_rc() {
 723
 724     console.log("Fetching RC services - timer:" + refreshCount_ecs)
 725
 726     if (refreshCount_rc < 0) {
 727         refreshCount_rc = -1
 728         return
 729     } else {
 730         refreshCount_rc = refreshCount_rc - 1
 731     }
 732     setTimeout(() => {
 733
 734         if (checkFunctionFlag("rc_stat")) {
 735             getSimCtr(LOCALHOST+RC_PORT+"/services", 0, function(data, index) {
 736                 var tmp_serv=""
 737                 try {
 738                     var jd=JSON.parse(data);
 739                     for(var i=0;i<jd.length;i++) {
 740                         if (tmp_serv.length > 0) {
 741                             tmp_serv=tmp_serv+","
 742                         }
 743                         tmp_serv=tmp_serv+jd[i]["name"]
 744                     }
 745
 746                 }
 747                 catch (err) {
 748                     tmp_serv="no_response"
 749                 }
 750                 rc_services=tmp_serv
 751             });
 752             clearFlag("rc_stat")
 753         }
 754         fetchAllMetrics_rc();
 755     }, 500)
 756 }
 757
 758 // Monitor for CR db
 759 app.get("/mon3",function(req, res){
 760
 761     console.log("Creating CR DB page - timer: " + refreshCount_ecs)
 762
 763     if (refreshCount_cr < 0) {
 764         refreshCount_cr=5
 765         fetchAllMetrics_cr()
 766     }
 767     refreshCount_cr=5
 768     var json_str=JSON.stringify(cr_db, null, 1)
 769     var htmlStr = "<!DOCTYPE html>" +
 770     "<html>" +
 771     "<head>" +
 772       "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 773       "<title>CR DB dump</title>"+
 774       "</head>" +
 775       "<body style=\"white-space: pre-wrap\">" +
 776       json_str +
 777       "</body>" +
 778       "</html>";
 779     res.send(htmlStr);
 780 })
 781
 782 // Monitor for ECS
 783 app.get("/mon2",function(req, res){
 784
 785     console.log("Creating enrichment metrics - timer: " + refreshCount_ecs)
 786
 787     if (refreshCount_ecs < 0) {
 788         refreshCount_ecs=5
 789         fetchAllMetrics_ecs()
 790     }
 791     refreshCount_ecs=5
 792
 793     var summary=req.query.summary
 794
 795     if (summary == undefined) {
 796         return res.redirect('/mon2?summary=false');
 797     }
 798
 799   //Build web page
 800         var htmlStr = "<!DOCTYPE html>" +
 801           "<html>" +
 802           "<head>" +
 803             "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 804             "<title>Enrichment coordinator service and producer stub</title>"+
 805             "</head>" +
 806             "<body>" +
 807             "<font size=\"-3\" face=\"summary\">"
 808             if (summary == "false") {
 809                 htmlStr=htmlStr+"<p>Set query param '?summary' to true to only show summary statistics.</p>"
 810             } else {
 811                 htmlStr=htmlStr+"<p>Set query param '?summary' to false to only show full statistics</p>"
 812             }
 813             if (ecs_job_status.length > 10) {
 814                 htmlStr=htmlStr+"<div style=\"color:red\"> Avoid running the server for large number of producers and/or jobs</div>"
 815             }
 816             htmlStr=htmlStr+"</font>" +
 817             "<h3>Enrichment Coordinator Service</h3>" +
 818             "<font face=\"monospace\">" +
 819             "Status:..........." + formatDataRow(ecs1) + "<br>" +
 820             "Producers:........" + formatDataRow(ecs2) + "<br>" +
 821             "Types:............" + formatDataRow(ecs3) + "<br>" +
 822             "Jobs:............." + formatDataRow(ecs4) + "<br>" +
 823             "</font>"
 824             if (summary == "false") {
 825                 htmlStr=htmlStr+
 826                 "<h4>Details</h4>" +
 827                 "<font face=\"monospace\">" +
 828                 "Producer ids:....." + formatDataRow(ecs_producers) + "<br>" +
 829                 "Type ids:........." + formatDataRow(ecs_types) + "<br>" +
 830                 "<br>";
 831                 for(var i=0;i<ecs_producer_type_arr.length;i++) {
 832                     var tmp=ecs_producer_type_arr[i]
 833                     if (tmp != undefined) {
 834                         var s = "Producer types...." + formatDataRow(ecs_producer_type_arr[i]) + "<br>"
 835                         htmlStr=htmlStr+s
 836                     }
 837                 }
 838                 htmlStr=htmlStr+"<br>";
 839                 for(i=0;i<ecs_producer_jobs_arr.length;i++) {
 840                     tmp=ecs_producer_jobs_arr[i]
 841                     if (tmp != undefined) {
 842                         s = "Producer jobs....." + formatDataRow(ecs_producer_jobs_arr[i]) + "<br>"
 843                         htmlStr=htmlStr+s
 844                     }
 845                 }
 846                 htmlStr=htmlStr+"<br>";
 847                 for(i=0;i<ecs_producer_status_arr.length;i++) {
 848                     tmp=ecs_producer_status_arr[i]
 849                     if (tmp != undefined) {
 850                         s = "Producer status..." + formatDataRow(tmp) + "<br>"
 851                         htmlStr=htmlStr+s
 852                     }
 853                 }
 854                 htmlStr=htmlStr+"<br>";
 855                 for(i=0;i<ecs_job_status.length;i++) {
 856                     tmp=ecs_job_status[i]
 857                     if (tmp != undefined) {
 858                         s = padding("Job", 18, ".") + formatDataRow(tmp) + "<br>"
 859                         htmlStr=htmlStr+s
 860                     }
 861                 }
 862                 htmlStr=htmlStr+"<br>"+"<br>" +
 863                 "</font>"
 864             }
 865
 866             htmlStr=htmlStr+
 867             "<h3>Producer stub</h3>" +
 868             "<font face=\"monospace\">" +
 869             "Producers:........" + formatDataRow(ps2) + "<br>" +
 870             "Types:............" + formatDataRow(ps3) + "<br>" +
 871             "Jobs:............." + formatDataRow(ps4) + "<br>" +
 872             "</font>"
 873             if (summary == "false") {
 874                 htmlStr=htmlStr+
 875                 "<h4>Details</h4>" +
 876                 "<font face=\"monospace\">" +
 877                 "Producer ids:....." + formatDataRow(ps_producers) + "<br>" +
 878                 "Type ids:........." + formatDataRow(ps_types) + "<br>" +
 879                 "<br>";
 880                 for(i=0;i<ps_producer_type_arr.length;i++) {
 881                     tmp=ps_producer_type_arr[i]
 882                     if (tmp != undefined) {
 883                         s = "Producer types...." + formatDataRow(ps_producer_type_arr[i]) + "<br>"
 884                         htmlStr=htmlStr+s
 885                     }
 886                 }
 887                 htmlStr=htmlStr+"<br>";
 888                 for(i=0;i<ps_producer_jobs_arr.length;i++) {
 889                     tmp=ps_producer_jobs_arr[i]
 890                     if (tmp != undefined) {
 891                         s = "Producer jobs....." + formatDataRow(ps_producer_jobs_arr[i]) + "<br>"
 892                         htmlStr=htmlStr+s
 893                     }
 894                 }
 895                 htmlStr=htmlStr+"<br>";
 896                 for(i=0;i<ps_producer_delivery_arr.length;i++) {
 897                     tmp=ps_producer_delivery_arr[i]
 898                     if (tmp != undefined) {
 899                         s = "Producer delivery." + formatDataRow(ps_producer_delivery_arr[i]) + "<br>"
 900                         htmlStr=htmlStr+s
 901                     }
 902                 }
 903             }
 904             htmlStr=htmlStr+
 905             "</font>" +
 906            "</body>" +
 907           "</html>";
 908         res.send(htmlStr);
 909 })
 910
 911 // Monitor for policy management
 912 app.get("/mon",function(req, res){
 913
 914     console.log("Creating policy metrics page " + refreshCount_pol)
 915
 916     if (refreshCount_pol < 0) {
 917         refreshCount_pol=5
 918         fetchAllMetrics_pol()
 919     }
 920     refreshCount_pol=5
 921
 922     if (refreshCount_rc < 0) {
 923         refreshCount_rc=5
 924         fetchAllMetrics_rc()
 925     }
 926     refreshCount_rc=5
 927
 928     var bn=req.query.basename
 929     pmsprefix=req.query.pmsprefix
 930
 931     console.log("PMS"+pmsprefix)
 932     if ((bn == undefined) || (pmsprefix == undefined)) {
 933         getCtr=0
 934         return res.redirect('/mon?basename=ricsim&pmsprefix=/a1-policy/v2');
 935     } else {
 936         ricbasename=bn
 937     }
 938
 939
 940     //Build web page
 941         var htmlStr = "<!DOCTYPE html>" +
 942           "<html>" +
 943           "<head>" +
 944             "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 945             "<title>Policy Management Service and simulator monitor</title>"+
 946             "</head>" +
 947             "<body>" +
 948             "<font size=\"-3\" face=\"monospace\">" +
 949             "<p>Change basename in url if other ric sim prefix is used</p>" +
 950             "<p>Change pmsprefix in url if pms with other prefix is used</p>" +
 951             "</font>" +
 952             "<h3>Policy Management Service</h3>" +
 953             "<font face=\"monospace\">" +
 954             "Status:..............................." + formatDataRow(ag1) + "<br>" +
 955             "Services:............................." + formatIdRowCompact(ag2) + "<br>" +
 956             "Types:................................" + formatIdRowCompact(ag3) + "<br>" +
 957             "Number of instances:.................." + formatDataRow(ag4) + "<br>" +
 958             "Near-RT RICs:........................." + formatDataRow(ag5) + "<br>" +
 959             "</font>" +
 960             "<h3>MR Stub interface</h3>" +
 961             "<font face=\"monospace\">"+
 962             "Submitted requests:............................" + formatDataRow(mr1) + "<br>" +
 963             "Fetched requests:.............................." + formatDataRow(mr2) + "<br>" +
 964             "Current requests waiting:......................" + formatDataRow(mr3) + "<br>" +
 965             "Submitted responses:..........................." + formatDataRow(mr4) + "<br>" +
 966             "Fetched responses.............................." + formatDataRow(mr5) + "<br>" +
 967             "Current responses waiting......................" + formatDataRow(mr6) + "<br>" +
 968             "</font>"+
 969             "<h3>Callback|Notification receiver</h3>" +
 970             "<font face=\"monospace\">" +
 971             "Callbacks received:..................." + formatDataRow(cr1) + "<br>" +
 972             "Callbacks fetched:...................." + formatDataRow(cr2) + "<br>" +
 973             "Number of waiting callback messages:.." + formatDataRow(cr3) + "<br>" +
 974             "</font>" +
 975             "<h3>R-APP Catalogue</h3>" +
 976             "<font face=\"monospace\">" +
 977             "Services:............................." + formatIdRowCompact(rc_services) + "<br>" +
 978             "</font>" +
 979             "<h3>Near-RT RIC | A1 Simulators</h3>" +
 980             "<font face=\"monospace\">"
 981
 982             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
 983             htmlStr=htmlStr+padding("Types", 10,"&nbsp;")
 984             htmlStr=htmlStr+padding("Instances", 12,"&nbsp;")
 985             htmlStr=htmlStr+padding("Data delivery", 12,"&nbsp;")+"<br>"
 986             htmlStr=htmlStr+padding("",70,"=")+"<br>"
 987             for(var simIndex=0;simIndex<simnames.length;simIndex++) {
 988                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
 989                 htmlStr=htmlStr+padding(simvar2[simIndex],10,"&nbsp;")
 990                 htmlStr=htmlStr+padding(simvar1[simIndex],12    ,"&nbsp;")
 991                 htmlStr=htmlStr+padding(simvar6[simIndex],12,"&nbsp;")
 992                 htmlStr=htmlStr+"<br>";
 993             }
 994
 995             htmlStr=htmlStr+"<br>";
 996             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
 997             htmlStr=htmlStr+padding("Version", 20,"&nbsp;")
 998             htmlStr=htmlStr+padding("Type-IDs", 10,"&nbsp;")+"<br>"
 999             htmlStr=htmlStr+padding("",65,"=")+"<br>"
1000             for(simIndex=0;simIndex<simnames.length;simIndex++) {
1001                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
1002                 htmlStr=htmlStr+padding(simvar4[simIndex],20,"&nbsp;")
1003                 htmlStr=htmlStr+padding(formatIdRowCompact(simvar3[simIndex]),10,"&nbsp;")
1004                 htmlStr=htmlStr+"<br>";
1005             }
1006
1007             htmlStr=htmlStr+"<br>";
1008             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
1009             htmlStr=htmlStr+padding("Remote hosts", 50,"&nbsp;")+"<br>"
1010             htmlStr=htmlStr+padding("",90,"=")+"<br>"
1011             for(simIndex=0;simIndex<simnames.length;simIndex++) {
1012                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
1013                 htmlStr=htmlStr+padding(simvar5[simIndex],50,"&nbsp;")
1014                 htmlStr=htmlStr+"<br>";
1015             }
1016
1017             htmlStr=htmlStr+
1018            "</body>" +
1019           "</html>";
1020         res.send(htmlStr);
1021 })
1022
1023 var httpServer = http.createServer(app);
1024 var httpPort=9999;
1025 httpServer.listen(httpPort);
1026 console.log("Simulator monitor listening (http) at "+httpPort);
1027 console.log("Open the web page on localhost:9999/mon to view the policy statistics page.")
1028 console.log("Open the web page on localhost:9999/mon2 to view the enrichment statistics page.")
1029 console.log("Open the web page on localhost:9999/mon3 to view CR DB in json.")