test/simulator-group/sim-monitor.js

   1 /*
   2 #  ============LICENSE_START===============================================
   3 #  Copyright (C) 2020 Nordix Foundation. All rights reserved.
   4 #  ========================================================================
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16 #  ============LICENSE_END=================================================
  17 #
  18 */
  19
  20 // Sim mon server - query the agent and the simulators for counters and other data
  21 // Presents a web page on localhost:9999/mon
  22
  23 var LOCALHOST="http://127.0.0.1:"
  24 var MRSTUB_PORT="3905"
  25 var AGENT_PORT="8081"
  26 var CR_PORT="8090"
  27 var ECS_PORT="8083"
  28 var PRODSTUB_PORT="8092"
  29
  30 var http = require('http');
  31
  32 var express = require('express');
  33 const { POINT_CONVERSION_HYBRID } = require('constants')
  34 var app = express();
  35 var fieldSize=32;
  36
  37 var flagstore={}
  38
  39 //I am alive
  40 app.get("/",function(req, res){
  41         res.send("ok");
  42 })
  43
  44 //Get parameter value from other server
  45 function getSimCtr(url, index, cb) {
  46     var data = '';
  47
  48     try {
  49         http.get(url, (resp) => {
  50             // A chunk of data has been recieved.
  51             resp.on('data', (chunk) => {
  52                 data += chunk;
  53             });
  54
  55             // The whole response has been received.
  56             resp.on('end', () => {
  57                 var code=resp.statusCode
  58                 if (code > 199 && code < 300) {
  59                     cb(data, index);
  60                 } else {
  61                     cb("not found", index);
  62                 }
  63             });
  64
  65         }).on("error", (err) => {
  66             console.log("Error: " + err.message);
  67             cb("no response", index);
  68         });
  69     } catch(err) {
  70         cb("no response", index);
  71     }
  72 }
  73
  74
  75 //Format a comma separated list of data to a html-safe string with fixed fieldsizes
  76 function formatDataRow(commaList) {
  77         var str = "";
  78         var tmp=commaList.split(',');
  79     for(var i=0;i<tmp.length;i++) {
  80         var data=tmp[i];
  81         var len = fieldSize-data.length;
  82         while(len>0) {
  83             data = data+"&nbsp;";
  84             len--;
  85         }
  86         str=str+data+"&nbsp;&nbsp;&nbsp;";
  87      }
  88         return str;
  89 }
  90
  91 //Format a comma separated list of ids to a html-safe string with fixed fieldsizes
  92 function formatIdRow(commaList) {
  93         var str = "";
  94         var tmp=commaList.split(',');
  95     for(var i=0;i<tmp.length;i++) {
  96         tmp[i] = tmp[i].trim();
  97         var data="&lt"+tmp[i]+"&gt";
  98         var len = fieldSize+4-data.length;
  99         while(len>0) {
 100             data = data+"&nbsp;";
 101             len--;
 102         }
 103         str=str+data+"&nbsp;&nbsp;&nbsp;";
 104     }
 105         return str;
 106 }
 107
 108 //Format a list of ids to a html-safe string in compact format
 109 function formatIdRowCompact(commaList) {
 110     if (commaList == undefined) {
 111         commaList= "";
 112     }
 113         var str = "";
 114         var tmp=commaList.split(',');
 115     for(var i=0;i<tmp.length;i++) {
 116         tmp[i] = tmp[i].trim();
 117         var data="&lt"+tmp[i]+"&gt";
 118         str=str+data+"&nbsp;";
 119     }
 120         return str;
 121 }
 122
 123 //Pad a string upto a certain size using a pad string
 124 function padding(val, size, pad) {
 125         var s=""+val;
 126         for(var i=s.length;i<size;i++) {
 127                 s=s+pad
 128         }
 129         return s;
 130 }
 131
 132 //Function to check if the previous call has returned, if so return true, if not return false
 133 //For preventing multiple calls to slow containers.
 134 function checkFunctionFlag(flag) {
 135     if (flagstore.hasOwnProperty(flag)) {
 136         if (flagstore[flag] == 0) {
 137             flagstore[flag]=1
 138             return true
 139         } else if (flagstore[flag] > 10) {
 140             //Reset flag after ten attempts
 141             console.log("Force release flag "+flag)
 142             flagstore[flag]=1
 143             return true
 144         } else {
 145             //Previous call not returned
 146             console.log("Flag not available "+flag)
 147             flagstore[flag]=flagstore[flag]+1
 148             return false
 149         }
 150     } else {
 151         flagstore[flag]=1
 152         return true
 153     }
 154 }
 155 //Clear flag for parameter
 156 function clearFlag(flag) {
 157     flagstore[flag]=0
 158 }
 159
 160 //Status variables, for parameters values fetched from other simulators
 161 var mr1="", mr2="", mr3="", mr4="", mr5="", mr6="";
 162
 163 //Status variables for agent
 164 var ag1=""
 165 var ag2=""
 166 var ag3=""
 167 var ag4=""
 168 var ag5=""
 169
 170 //Status variables for callback receiver
 171 var cr1=""
 172 var cr2=""
 173 var cr3=""
 174
 175
 176 //Container names and ports of the ric simulator
 177 var simnames=[]
 178 var simports=[]
 179
 180 //Status variables for each ric simulator
 181 var simvar1=[]
 182 var simvar2=[]
 183 var simvar3=[]
 184 var simvar4=[]
 185 var simvar5=[]
 186 var simvar6=[]
 187
 188 //Status variables, for parameters values fetched from ecs
 189 var ecs1="", ecs2="", ecs3="", ecs4="", ecs_types="-", ecs_producers="-";
 190 var ecs_producer_arr=new Array(0)
 191 var ecs_producer_type_arr=new Array(0)
 192 var ecs_producer_jobs_arr=new Array(0)
 193 var ecs_producer_status_arr=new Array(0)
 194 var ecs_jobs=new Array(0)
 195 var ecs_job_status=new Array(0)
 196
 197 //Status variables, for parameters values fetched from prodstub
 198 var ps2="", ps3="", ps4="", ps_types="-", ps_producers="-";
 199 var ps_producer_type_arr=new Array(0)
 200 var ps_producer_jobs_arr=new Array(0)
 201 var ps_producer_delivery_arr=new Array(0)
 202
 203 //Full CR DB
 204 var cr_db={}
 205
 206 //Counts the number of get request for the html page
 207 var getCtr=0
 208
 209 var refreshCount_pol=-1
 210
 211 var refreshCount_ecs=-1
 212
 213 var refreshCount_cr=-1
 214
 215 var ricbasename="ricsim"
 216
 217 function fetchAllMetrics_pol() {
 218
 219     console.log("Fetching policy metrics " + refreshCount_pol)
 220
 221     if (refreshCount_pol < 0) {
 222         refreshCount_pol = -1
 223         return
 224     } else {
 225         refreshCount_pol = refreshCount_pol - 1
 226     }
 227     setTimeout(() => {
 228
 229         if (getCtr%3 == 0) {
 230             //Extract the port numbers from the running simulators, for every 3 calls
 231             const { exec } = require('child_process');
 232             exec('docker ps --filter "name='+ricbasename+'" --format "{{.Names}} {{.Ports}}" | sed s/0.0.0.0:// | cut -d \'>\' -f1 | sed \'s/[[-]]*$//\'', (err, stdout, stderr) => {
 233
 234                 var simulators = ""
 235                 simulators=`${stdout}`.replace(/(\r\n|\n|\r)/gm," ");
 236                 simulators=simulators.trim();
 237                 var sims=simulators.split(" ")
 238                 simnames=[]
 239                 simports=[]
 240                 for(var i=0;i<sims.length;i=i+2) {
 241                     simnames[i/2]=sims[i]
 242                     simports[i/2]=sims[i+1]
 243                 }
 244             });
 245         }
 246         getCtr=getCtr+1
 247
 248         //Get metric values from the simulators
 249         for(var index=0;index<simnames.length;index++) {
 250
 251             if (checkFunctionFlag("simvar1_"+index)) {
 252                 getSimCtr(LOCALHOST+simports[index]+"/counter/num_instances", index, function(data, idx) {
 253                     simvar1[idx] = data;
 254                     clearFlag("simvar1_"+idx)
 255                 });
 256             }
 257             if (checkFunctionFlag("simvar2_"+index)) {
 258                 getSimCtr(LOCALHOST+simports[index]+"/counter/num_types", index, function(data,idx) {
 259                     simvar2[idx] = data;
 260                     clearFlag("simvar2_"+idx)
 261                 });
 262             }
 263             if (checkFunctionFlag("simvar3_"+index)) {
 264                 getSimCtr(LOCALHOST+simports[index]+"/policytypes", index, function(data,idx) {
 265                     data=data.replace(/\[/g,'');
 266                     data=data.replace(/\]/g,'');
 267                     data=data.replace(/ /g,'');
 268                     data=data.replace(/\"/g,'');
 269                     simvar3[idx] = data;
 270                     clearFlag("simvar3_"+idx)
 271                 });
 272             }
 273             if (checkFunctionFlag("simvar4_"+index)) {
 274                 getSimCtr(LOCALHOST+simports[index]+"/counter/interface", index, function(data,idx) {
 275                     simvar4[idx] = data;
 276                     clearFlag("simvar4_"+idx)
 277                 });
 278             }
 279             if (checkFunctionFlag("simvar5_"+index)) {
 280                 getSimCtr(LOCALHOST+simports[index]+"/counter/remote_hosts", index, function(data,idx) {
 281                     simvar5[idx] = data;
 282                     clearFlag("simvar5_"+idx)
 283                 });
 284             }
 285             if (checkFunctionFlag("simvar6_"+index)) {
 286                 getSimCtr(LOCALHOST+simports[index]+"/counter/datadelivery", index, function(data,idx) {
 287                     simvar6[idx] = data;
 288                     clearFlag("simvar6_"+idx)
 289                 });
 290             }
 291         }
 292
 293         //MR - get metrics values from the MR stub
 294         if (checkFunctionFlag("mr1")) {
 295             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/requests_submitted", 0, function(data, idx) {
 296                 mr1 = data;
 297                 clearFlag("mr1")
 298             });
 299         }
 300         if (checkFunctionFlag("mr2")) {
 301             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/requests_fetched", 0, function(data, idx) {
 302                 mr2 = data;
 303                 clearFlag("mr2")
 304             });
 305         }
 306         if (checkFunctionFlag("mr3")) {
 307             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/current_requests", 0, function(data, idx) {
 308                 mr3 = data;
 309                 clearFlag("mr3")
 310             });
 311         }
 312         if (checkFunctionFlag("mr4")) {
 313             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/responses_submitted", 0, function(data, idx) {
 314                 mr4 = data;
 315                 clearFlag("mr4")
 316             });
 317         }
 318         if (checkFunctionFlag("mr5")) {
 319             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/responses_fetched", 0, function(data, idx) {
 320                 mr5 = data;
 321                 clearFlag("mr5")
 322             });
 323         }
 324         if (checkFunctionFlag("mr6")) {
 325             getSimCtr(LOCALHOST+MRSTUB_PORT+"/counter/current_responses", 0, function(data, idx) {
 326                 mr6 = data;
 327                 clearFlag("mr6")
 328             });
 329         }
 330
 331         //CR - get metrics values from the callbackreceiver
 332         if (checkFunctionFlag("cr1")) {
 333             getSimCtr(LOCALHOST+CR_PORT+"/counter/received_callbacks", 0, function(data, idx) {
 334                 cr1 = data;
 335                 clearFlag("cr1")
 336             });
 337         }
 338         if (checkFunctionFlag("cr2")) {
 339             getSimCtr(LOCALHOST+CR_PORT+"/counter/fetched_callbacks", 0, function(data, idx) {
 340                 cr2 = data;
 341                 clearFlag("cr2")
 342             });
 343         }
 344         if (checkFunctionFlag("cr3")) {
 345             getSimCtr(LOCALHOST+CR_PORT+"/counter/current_messages", 0, function(data, idx) {
 346                 cr3 = data;
 347                 clearFlag("cr3")
 348             });
 349         }
 350         //Agent - more get metrics from the agent
 351         if (checkFunctionFlag("ag1")) {
 352             getSimCtr(LOCALHOST+AGENT_PORT+"/status", 0, function(data, idx) {
 353                 ag1 = data;
 354                 clearFlag("ag1")
 355             });
 356         }
 357         if (checkFunctionFlag("ag2")) {
 358             getSimCtr(LOCALHOST+AGENT_PORT+"/services", 0, function(data, idx) {
 359                 ag2="";
 360                 try {
 361                     var jd=JSON.parse(data);
 362                     for(var key in jd) {
 363                         if (ag2.length > 1) {
 364                             ag2=ag2+", "
 365                         }
 366                         ag2=ag2+(jd[key]["serviceName"]).trim()
 367                     }
 368                 }
 369                 catch (err) {
 370                     ag2=data
 371                 }
 372                 clearFlag("ag2")
 373             });
 374         }
 375         if (checkFunctionFlag("ag3")) {
 376             getSimCtr(LOCALHOST+AGENT_PORT+"/policy_types", 0, function(data, idx) {
 377                 ag3="";
 378                 try {
 379                     var jd=JSON.parse(data);
 380                     for(var key in jd) {
 381                         if (ag3.length > 0) {
 382                             ag3=ag3+", "
 383                         }
 384                         ag3=ag3+jd[key].trim()
 385                     }
 386                 }
 387                 catch (err) {
 388                     ag3=""
 389                 }
 390                 clearFlag("ag3")
 391             });
 392         }
 393
 394         if (checkFunctionFlag("ag4")) {
 395             getSimCtr(LOCALHOST+AGENT_PORT+"/policy_ids", 0, function(data, idx) {
 396                 try {
 397                     var jd=JSON.parse(data);
 398                     ag4=""+jd.length
 399                 }
 400                 catch (err) {
 401                     ag4=""
 402                 }
 403                 clearFlag("ag4")
 404             });
 405         }
 406
 407         if (checkFunctionFlag("ag5")) {
 408             getSimCtr(LOCALHOST+AGENT_PORT+"/rics", 0, function(data, idx) {
 409                 try {
 410                     var jd=JSON.parse(data);
 411                     ag5=""+jd.length
 412                 }
 413                 catch (err) {
 414                     ag5=""
 415                 }
 416                 clearFlag("ag5")
 417             });
 418         }
 419
 420         fetchAllMetrics_pol();
 421
 422     }, 500)
 423 }
 424
 425 function fetchAllMetrics_ecs() {
 426
 427     console.log("Fetching enrichment metrics - timer:" + refreshCount_ecs)
 428
 429     if (refreshCount_ecs < 0) {
 430         refreshCount_ecs = -1
 431         return
 432     } else {
 433         refreshCount_ecs = refreshCount_ecs - 1
 434     }
 435     setTimeout(() => {
 436
 437         if (checkFunctionFlag("ecs_stat")) {
 438             getSimCtr(LOCALHOST+ECS_PORT+"/status", 0, function(data, index) {
 439                 try {
 440                     var jd=JSON.parse(data);
 441                     ecs1=jd["status"]
 442                     ecs2=""+jd["no_of_producers"]
 443                     ecs3=""+jd["no_of_types"]
 444                     ecs4=""+jd["no_of_jobs"]
 445                 }
 446                 catch (err) {
 447                     ecs1="error response"
 448                     ecs2="error response"
 449                     ecs3="error response"
 450                     ecs4="error response"
 451                 }
 452             });
 453             clearFlag("ecs_stat")
 454         }
 455         if (checkFunctionFlag("ecs_types")) {
 456             getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eitypes", 0, function(data, index) {
 457                 var tmp_ecs_types="-"
 458                 try {
 459                     var jd=JSON.parse(data);
 460                     for(var i=0;i<jd.length;i++) {
 461                         if (tmp_ecs_types.length == 1) {
 462                             tmp_ecs_types=""
 463                         }
 464                         tmp_ecs_types=""+tmp_ecs_types+jd[i]+" "
 465                     }
 466                 }
 467                 catch (err) {
 468                     tmp_ecs_types="error response"
 469                 }
 470                 ecs_types = tmp_ecs_types
 471             });
 472             clearFlag("ecs_types")
 473         }
 474         if (checkFunctionFlag("ecs_producers")) {
 475             getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers", 0, function(data, index) {
 476                 var tmp_ecs_producers="-"
 477                 try {
 478                     var jd=JSON.parse(data);
 479                     var tmp_ecs_producer_arr=new Array(jd.length)
 480                     for(var i=0;i<jd.length;i++) {
 481                         if (tmp_ecs_producers.length == 1) {
 482                             tmp_ecs_producers=""
 483                         }
 484                         tmp_ecs_producers=""+tmp_ecs_producers+jd[i]+" "
 485                         tmp_ecs_producer_arr[i]=jd[i]
 486                     }
 487                     ecs_producer_arr = tmp_ecs_producer_arr
 488                     ecs_producers = tmp_ecs_producers
 489                 }
 490                 catch (err) {
 491                     ecs_producers="error response"
 492                     ecs_producer_arr=new Array(0)
 493                 }
 494             });
 495             clearFlag("ecs_producers")
 496         }
 497         if (checkFunctionFlag("ecs_data")) {
 498             try {
 499                 var tmp_ecs_producer_type_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 500                 for(var x=0;x<tmp_ecs_producer_type_arr.length;x++) {
 501                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_type_arr[x], x, function(data, idx) {
 502                         var row=""+tmp_ecs_producer_type_arr[idx]+" : "
 503                         try {
 504                             var jd=JSON.parse(data);
 505                             var jda=jd["supported_ei_types"]
 506                             for(var j=0;j<jda.length;j++) {
 507                                 row=""+row+jda[j]["ei_type_identity"]+" "
 508                             }
 509                             tmp_ecs_producer_type_arr[idx]=row
 510                         }
 511                         catch (err) {
 512                             tmp_ecs_producer_type_arr=new Array(0)
 513                         }
 514                     });
 515                 }
 516                 ecs_producer_type_arr = tmp_ecs_producer_type_arr
 517             } catch (err) {
 518                 ecs_producer_type_arr=new Array(0)
 519             }
 520             try {
 521                 var tmp_ecs_producer_jobs_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 522                 for(x=0;x<tmp_ecs_producer_jobs_arr.length;x++) {
 523                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_jobs_arr[x]+"/eijobs", x, function(data, idx) {
 524                         var row=""+tmp_ecs_producer_jobs_arr[idx]+" : "
 525                         try {
 526                             var jd=JSON.parse(data);
 527                             for(var j=0;j<jd.length;j++) {
 528                                 var jda=jd[j]
 529                                 row=""+row+jda["ei_job_identity"]+"("+jda["ei_type_identity"]+") "
 530                             }
 531                             tmp_ecs_producer_jobs_arr[idx]=row
 532                         }
 533                         catch (err) {
 534                             tmp_ecs_producer_jobs_arr=new Array(0)
 535                         }
 536                     });
 537                 }
 538                 ecs_producer_jobs_arr = tmp_ecs_producer_jobs_arr
 539             } catch (err) {
 540                 ecs_producer_jobs_arr=new Array(0)
 541             }
 542
 543             try {
 544                 var tmp_ecs_producer_status_arr = JSON.parse(JSON.stringify(ecs_producer_arr))
 545                 for(x=0;x<tmp_ecs_producer_status_arr.length;x++) {
 546                     getSimCtr(LOCALHOST+ECS_PORT+"/ei-producer/v1/eiproducers/"+tmp_ecs_producer_status_arr[x]+"/status", x, function(data, idx) {
 547                         var row=""+tmp_ecs_producer_status_arr[idx]+" : "
 548                         try {
 549                             var jd=JSON.parse(data);
 550                             row=""+row+jd["operational_state"]
 551                             tmp_ecs_producer_status_arr[idx]=row
 552                         }
 553                         catch (err) {
 554                             tmp_ecs_producer_status_arr=new Array(0)
 555                         }
 556                     });
 557                 }
 558                 ecs_producer_status_arr = tmp_ecs_producer_status_arr
 559             } catch (err) {
 560                 ecs_producer_status_arr=new Array(0)
 561             }
 562             clearFlag("ecs_data")
 563         }
 564         if (checkFunctionFlag("ecs_jobs")) {
 565             getSimCtr(LOCALHOST+ECS_PORT+"/A1-EI/v1/eijobs", 0, function(data, index) {
 566                 try {
 567                     var jd=JSON.parse(data);
 568                     var tmpArr=new Array(jd.length)
 569                     for(var i=0;i<jd.length;i++) {
 570                         tmpArr[i]=jd[i]
 571                     }
 572                     ecs_jobs=tmpArr
 573                 }
 574                 catch (err) {
 575                     ecs_jobs=new Array(0)
 576                 }
 577             });
 578             clearFlag("ecs_jobs")
 579         }
 580         if (checkFunctionFlag("ecs_job_status")) {
 581             try {
 582                 var tmp_ecs_job_status= JSON.parse(JSON.stringify(ecs_jobs))
 583                 for(x=0;x<tmp_ecs_job_status.length;x++) {
 584                     getSimCtr(LOCALHOST+ECS_PORT+"/A1-EI/v1/eijobs/"+tmp_ecs_job_status[x]+"/status", x, function(data, idx) {
 585                         try {
 586                             var jd=JSON.parse(data);
 587                             tmp_ecs_job_status[idx]=""+tmp_ecs_job_status[idx]+":"+jd["eiJobStatus"]
 588                         }
 589                         catch (err) {
 590                             tmp_ecs_job_status="-"
 591                         }
 592                     });
 593                 }
 594                 ecs_job_status = tmp_ecs_job_status
 595             } catch (err) {
 596                 ecs_job_status="-"
 597             }
 598             clearFlag("ecs_job_status")
 599         }
 600         if (checkFunctionFlag("prodstub_stat")) {
 601             getSimCtr(LOCALHOST+PRODSTUB_PORT+"/status", x, function(data, idx) {
 602                 var ctr2_map=new Map()
 603                 var ctr3_map=new Map()
 604                 var ctr2=0
 605                 var ctr4=0
 606                 var tmp_ps_producers=""
 607                 var tmp_ps_types=""
 608                 var tmp_ps_producer_type_arr=new Array()
 609                 var tmp_ps_producer_jobs_arr=new Array()
 610                 var tmp_ps_producer_delivery_arr=new Array()
 611                 var tmp_ps2=""
 612                 var tmp_ps3=""
 613                 var tmp_ps4=""
 614                 try {
 615                     var jp=JSON.parse(data);
 616                     for(var prod_name in jp) {
 617                         ctr2_map.set(prod_name, prod_name)
 618                         ctr2 += 1
 619                         var jj=jp[prod_name]
 620                         var row=""+prod_name+" : "
 621                         var rowj=""+prod_name+" : "
 622                         var rowd=""+prod_name+" : "
 623                         tmp_ps_producers += prod_name + " "
 624                         for(var ji in jj) {
 625                             if (ji == "types") {
 626                                 var ta=jj[ji]
 627                                 for(var i=0;i<ta.length;i++) {
 628                                     ctr3_map.set(ta[i], ta[i])
 629                                     row += " "+ta[i]
 630                                 }
 631                             } else if (ji == "supervision_response") {
 632                                 //Do nothing
 633                             } else if (ji == "supervision_counter") {
 634                                 //Do nothing
 635                             } else if (ji == "types") {
 636                                 //Do nothing
 637                             } else {
 638                                 ctr4 += 1
 639                                 rowj += " "+ji
 640                                 rowd += " "+ji
 641                                 var job_data=jj[ji]["json"]
 642                                 if (job_data != undefined) {
 643                                     rowj += "("+job_data["ei_type_identity"]+")"
 644                                 }
 645                                 rowd += "("+jj[ji]["delivery_attempts"]+")"
 646                             }
 647                         }
 648                         tmp_ps_producer_type_arr[(ctr2-1)]=row
 649                         tmp_ps_producer_jobs_arr[(ctr2-1)]=rowj
 650                         tmp_ps_producer_delivery_arr[(ctr2-1)]=rowd
 651                     }
 652                     tmp_ps2=""+ctr2_map.size
 653                     tmp_ps3=""+ctr3_map.size
 654                     for(const [key, value] of ctr3_map.entries()) {
 655                         tmp_ps_types += key + " "
 656                     }
 657                     tmp_ps4=""+ctr4
 658
 659                     ps_producers=tmp_ps_producers
 660                     ps_types=tmp_ps_types
 661                     ps_producer_type_arr=tmp_ps_producer_type_arr
 662                     ps_producer_jobs_arr=tmp_ps_producer_jobs_arr
 663                     ps_producer_delivery_arr=tmp_ps_producer_delivery_arr
 664                     ps2=tmp_ps2
 665                     ps3=tmp_ps3
 666                     ps4=tmp_ps4
 667                 }
 668                 catch (err) {
 669                     ps_producers="error response"
 670                     ps_types="error response"
 671                     ps_producer_type_arr=new Array()
 672                     ps_producer_jobs_arr=new Array()
 673                     ps_producer_delivery_arr=new Array()
 674                     ps2="error response"
 675                     ps3="error response"
 676                     ps4="error response"
 677                 }
 678             });
 679             clearFlag("prodstub_stat")
 680         }
 681
 682         fetchAllMetrics_ecs();
 683
 684     }, 500)
 685 }
 686
 687 function fetchAllMetrics_cr() {
 688
 689     console.log("Fetching CR DB - timer:" + refreshCount_ecs)
 690
 691     if (refreshCount_cr < 0) {
 692         refreshCount_cr = -1
 693         return
 694     } else {
 695         refreshCount_cr = refreshCount_cr - 1
 696     }
 697     setTimeout(() => {
 698
 699         if (checkFunctionFlag("cr_stat")) {
 700             getSimCtr(LOCALHOST+CR_PORT+"/db", 0, function(data, index) {
 701                 try {
 702                     cr_db=JSON.parse(data);
 703                 }
 704                 catch (err) {
 705                     cr_db={}
 706                 }
 707             });
 708             clearFlag("cr_stat")
 709         }
 710         fetchAllMetrics_cr();
 711     }, 500)
 712 }
 713
 714 // Monitor for CR db
 715 app.get("/mon3",function(req, res){
 716
 717     console.log("Creating CR DB page - timer: " + refreshCount_ecs)
 718
 719     if (refreshCount_cr < 0) {
 720         refreshCount_cr=5
 721         fetchAllMetrics_cr()
 722     }
 723     refreshCount_cr=5
 724     var json_str=JSON.stringify(cr_db, null, 1)
 725     var htmlStr = "<!DOCTYPE html>" +
 726     "<html>" +
 727     "<head>" +
 728       "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 729       "<title>CR DB dump</title>"+
 730       "</head>" +
 731       "<body style=\"white-space: pre-wrap\">" +
 732       json_str +
 733       "</body>" +
 734       "</html>";
 735     res.send(htmlStr);
 736 })
 737
 738 // Monitor for ECS
 739 app.get("/mon2",function(req, res){
 740
 741     console.log("Creating enrichment metrics - timer: " + refreshCount_ecs)
 742
 743     if (refreshCount_ecs < 0) {
 744         refreshCount_ecs=5
 745         fetchAllMetrics_ecs()
 746     }
 747     refreshCount_ecs=5
 748
 749     var summary=req.query.summary
 750
 751     if (summary == undefined) {
 752         return res.redirect('/mon2?summary=false');
 753     }
 754
 755   //Build web page
 756         var htmlStr = "<!DOCTYPE html>" +
 757           "<html>" +
 758           "<head>" +
 759             "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 760             "<title>Enrichment coordinator service and producer stub</title>"+
 761             "</head>" +
 762             "<body>" +
 763             "<font size=\"-3\" face=\"summary\">"
 764             if (summary == "false") {
 765                 htmlStr=htmlStr+"<p>Set query param '?summary' to true to only show summary statistics.</p>"
 766             } else {
 767                 htmlStr=htmlStr+"<p>Set query param '?summary' to false to only show full statistics</p>"
 768             }
 769             if (ecs_job_status.length > 10) {
 770                 htmlStr=htmlStr+"<div style=\"color:red\"> Avoid running the server for large number of producers and/or jobs</div>"
 771             }
 772             htmlStr=htmlStr+"</font>" +
 773             "<h3>Enrichment Coordinator Service</h3>" +
 774             "<font face=\"monospace\">" +
 775             "Status:..........." + formatDataRow(ecs1) + "<br>" +
 776             "Producers:........" + formatDataRow(ecs2) + "<br>" +
 777             "Types:............" + formatDataRow(ecs3) + "<br>" +
 778             "Jobs:............." + formatDataRow(ecs4) + "<br>" +
 779             "</font>"
 780             if (summary == "false") {
 781                 htmlStr=htmlStr+
 782                 "<h4>Details</h4>" +
 783                 "<font face=\"monospace\">" +
 784                 "Producer ids:....." + formatDataRow(ecs_producers) + "<br>" +
 785                 "Type ids:........." + formatDataRow(ecs_types) + "<br>" +
 786                 "<br>";
 787                 for(var i=0;i<ecs_producer_type_arr.length;i++) {
 788                     var tmp=ecs_producer_type_arr[i]
 789                     if (tmp != undefined) {
 790                         var s = "Producer types...." + formatDataRow(ecs_producer_type_arr[i]) + "<br>"
 791                         htmlStr=htmlStr+s
 792                     }
 793                 }
 794                 htmlStr=htmlStr+"<br>";
 795                 for(i=0;i<ecs_producer_jobs_arr.length;i++) {
 796                     tmp=ecs_producer_jobs_arr[i]
 797                     if (tmp != undefined) {
 798                         s = "Producer jobs....." + formatDataRow(ecs_producer_jobs_arr[i]) + "<br>"
 799                         htmlStr=htmlStr+s
 800                     }
 801                 }
 802                 htmlStr=htmlStr+"<br>";
 803                 for(i=0;i<ecs_producer_status_arr.length;i++) {
 804                     tmp=ecs_producer_status_arr[i]
 805                     if (tmp != undefined) {
 806                         s = "Producer status..." + formatDataRow(tmp) + "<br>"
 807                         htmlStr=htmlStr+s
 808                     }
 809                 }
 810                 htmlStr=htmlStr+"<br>";
 811                 for(i=0;i<ecs_job_status.length;i++) {
 812                     tmp=ecs_job_status[i]
 813                     console.log("tmp")
 814                     if (tmp != undefined) {
 815                         s = padding("Job", 18, ".") + formatDataRow(tmp) + "<br>"
 816                         htmlStr=htmlStr+s
 817                     }
 818                 }
 819                 htmlStr=htmlStr+"<br>"+"<br>" +
 820                 "</font>"
 821             }
 822
 823             htmlStr=htmlStr+
 824             "<h3>Producer stub</h3>" +
 825             "<font face=\"monospace\">" +
 826             "Producers:........" + formatDataRow(ps2) + "<br>" +
 827             "Types:............" + formatDataRow(ps3) + "<br>" +
 828             "Jobs:............." + formatDataRow(ps4) + "<br>" +
 829             "</font>"
 830             if (summary == "false") {
 831                 htmlStr=htmlStr+
 832                 "<h4>Details</h4>" +
 833                 "<font face=\"monospace\">" +
 834                 "Producer ids:....." + formatDataRow(ps_producers) + "<br>" +
 835                 "Type ids:........." + formatDataRow(ps_types) + "<br>" +
 836                 "<br>";
 837                 for(i=0;i<ps_producer_type_arr.length;i++) {
 838                     tmp=ps_producer_type_arr[i]
 839                     if (tmp != undefined) {
 840                         s = "Producer types...." + formatDataRow(ps_producer_type_arr[i]) + "<br>"
 841                         htmlStr=htmlStr+s
 842                     }
 843                 }
 844                 htmlStr=htmlStr+"<br>";
 845                 for(i=0;i<ps_producer_jobs_arr.length;i++) {
 846                     tmp=ps_producer_jobs_arr[i]
 847                     if (tmp != undefined) {
 848                         s = "Producer jobs....." + formatDataRow(ps_producer_jobs_arr[i]) + "<br>"
 849                         htmlStr=htmlStr+s
 850                     }
 851                 }
 852                 htmlStr=htmlStr+"<br>";
 853                 for(i=0;i<ps_producer_delivery_arr.length;i++) {
 854                     tmp=ps_producer_delivery_arr[i]
 855                     if (tmp != undefined) {
 856                         s = "Producer delivery." + formatDataRow(ps_producer_delivery_arr[i]) + "<br>"
 857                         htmlStr=htmlStr+s
 858                     }
 859                 }
 860             }
 861             htmlStr=htmlStr+
 862             "</font>" +
 863            "</body>" +
 864           "</html>";
 865         res.send(htmlStr);
 866 })
 867
 868 // Monitor for policy management
 869 app.get("/mon",function(req, res){
 870
 871     console.log("Creating policy metrics page " + refreshCount_pol)
 872
 873     if (refreshCount_pol < 0) {
 874         refreshCount_pol=5
 875         fetchAllMetrics_pol()
 876     }
 877     refreshCount_pol=5
 878
 879     var bn=req.query.basename
 880
 881     if (bn == undefined) {
 882         getCtr=0
 883         return res.redirect('/mon?basename=ricsim');
 884     } else {
 885         ricbasename=bn
 886     }
 887
 888     //Build web page
 889         var htmlStr = "<!DOCTYPE html>" +
 890           "<html>" +
 891           "<head>" +
 892             "<meta http-equiv=\"refresh\" content=\"2\">"+  //2 sec auto refresh
 893             "<title>Policy Agent and simulator monitor</title>"+
 894             "</head>" +
 895             "<body>" +
 896             "<font size=\"-3\" face=\"monospace\">" +
 897             "<p>Change basename in url if other ric sim prefix is used</p>" +
 898             "</font>" +
 899             "<h3>Policy agent</h3>" +
 900             "<font face=\"monospace\">" +
 901             "Status:..............................." + formatDataRow(ag1) + "<br>" +
 902             "Services:............................." + formatIdRowCompact(ag2) + "<br>" +
 903             "Types:................................" + formatIdRowCompact(ag3) + "<br>" +
 904             "Number of instances:.................." + formatDataRow(ag4) + "<br>" +
 905             "Near-RT RICs:........................." + formatDataRow(ag5) + "<br>" +
 906             "</font>" +
 907             "<h3>MR Stub interface</h3>" +
 908             "<font face=\"monospace\">"+
 909             "Submitted requests:............................" + formatDataRow(mr1) + "<br>" +
 910             "Fetched requests:.............................." + formatDataRow(mr2) + "<br>" +
 911             "Current requests waiting:......................" + formatDataRow(mr3) + "<br>" +
 912             "Submitted responses:..........................." + formatDataRow(mr4) + "<br>" +
 913             "Fetched responses.............................." + formatDataRow(mr5) + "<br>" +
 914             "Current responses waiting......................" + formatDataRow(mr6) + "<br>" +
 915             "</font>"+
 916             "<h3>Callback receiver</h3>" +
 917             "<font face=\"monospace\">" +
 918             "Callbacks received:..................." + formatDataRow(cr1) + "<br>" +
 919             "Callbacks fetched:...................." + formatDataRow(cr2) + "<br>" +
 920             "Number of waiting callback messages:.." + formatDataRow(cr3) + "<br>" +
 921             "</font>" +
 922             "<h3>Near-RT RIC Simulators</h3>" +
 923             "<font face=\"monospace\">"
 924
 925             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
 926             htmlStr=htmlStr+padding("Types", 10,"&nbsp;")
 927             htmlStr=htmlStr+padding("Instances", 12,"&nbsp;")
 928             htmlStr=htmlStr+padding("Data delivery", 12,"&nbsp;")+"<br>"
 929             htmlStr=htmlStr+padding("",70,"=")+"<br>"
 930             for(var simIndex=0;simIndex<simnames.length;simIndex++) {
 931                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
 932                 htmlStr=htmlStr+padding(simvar2[simIndex],10,"&nbsp;")
 933                 htmlStr=htmlStr+padding(simvar1[simIndex],12    ,"&nbsp;")
 934                 htmlStr=htmlStr+padding(simvar6[simIndex],12,"&nbsp;")
 935                 htmlStr=htmlStr+"<br>";
 936             }
 937
 938             htmlStr=htmlStr+"<br>";
 939             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
 940             htmlStr=htmlStr+padding("Version", 20,"&nbsp;")
 941             htmlStr=htmlStr+padding("Type-IDs", 10,"&nbsp;")+"<br>"
 942             htmlStr=htmlStr+padding("",65,"=")+"<br>"
 943             for(simIndex=0;simIndex<simnames.length;simIndex++) {
 944                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
 945                 htmlStr=htmlStr+padding(simvar4[simIndex],20,"&nbsp;")
 946                 htmlStr=htmlStr+padding(formatIdRowCompact(simvar3[simIndex]),10,"&nbsp;")
 947                 htmlStr=htmlStr+"<br>";
 948             }
 949
 950             htmlStr=htmlStr+"<br>";
 951             htmlStr=htmlStr+padding("Near-RT RIC Simulator name", 35,"&nbsp;")
 952             htmlStr=htmlStr+padding("Remote hosts", 50,"&nbsp;")+"<br>"
 953             htmlStr=htmlStr+padding("",90,"=")+"<br>"
 954             for(simIndex=0;simIndex<simnames.length;simIndex++) {
 955                 htmlStr=htmlStr+padding(simnames[simIndex]+ " ("+simports[simIndex]+")",35,"&nbsp;");
 956                 htmlStr=htmlStr+padding(simvar5[simIndex],50,"&nbsp;")
 957                 htmlStr=htmlStr+"<br>";
 958             }
 959
 960             htmlStr=htmlStr+
 961            "</body>" +
 962           "</html>";
 963         res.send(htmlStr);
 964 })
 965
 966 var httpServer = http.createServer(app);
 967 var httpPort=9999;
 968 httpServer.listen(httpPort);
 969 console.log("Simulator monitor listening (http) at "+httpPort);
 970 console.log("Open the web page on localhost:9999/mon to view the policy statistics page.")
 971 console.log("Open the web page on localhost:9999/mon2 to view the enrichment statistics page.")
 972 console.log("Open the web page on localhost:9999/mon3 to view CR DB in json.")