pmproducer/src/test/java/org/oran/pmproducer/IntegrationWithKafka.java

   1 /*-
   2  * ========================LICENSE_START=================================
   3  * O-RAN-SC
   4  * %%
   5  * Copyright (C) 2023 Nordix Foundation
   6  * %%
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ========================LICENSE_END===================================
  19  */
  20
  21 package org.oran.pmproducer;
  22
  23 import static org.assertj.core.api.Assertions.assertThat;
  24 import static org.awaitility.Awaitility.await;
  25
  26 import com.google.gson.JsonParser;
  27
  28 import java.io.IOException;
  29 import java.lang.invoke.MethodHandles;
  30 import java.nio.file.Path;
  31 import java.time.Instant;
  32 import java.time.OffsetDateTime;
  33 import java.util.ArrayList;
  34 import java.util.HashMap;
  35 import java.util.Map;
  36
  37 import lombok.Builder;
  38
  39 import org.apache.kafka.clients.producer.ProducerConfig;
  40 import org.apache.kafka.clients.producer.ProducerRecord;
  41 import org.apache.kafka.common.serialization.ByteArraySerializer;
  42 import org.junit.jupiter.api.AfterEach;
  43 import org.junit.jupiter.api.BeforeEach;
  44 import org.junit.jupiter.api.Test;
  45 import org.oran.pmproducer.clients.AsyncRestClient;
  46 import org.oran.pmproducer.clients.AsyncRestClientFactory;
  47 import org.oran.pmproducer.clients.SecurityContext;
  48 import org.oran.pmproducer.configuration.ApplicationConfig;
  49 import org.oran.pmproducer.configuration.WebClientConfig;
  50 import org.oran.pmproducer.configuration.WebClientConfig.HttpProxyConfig;
  51 import org.oran.pmproducer.controllers.ProducerCallbacksController;
  52 import org.oran.pmproducer.controllers.ProducerCallbacksController.StatisticsCollection;
  53 import org.oran.pmproducer.datastore.DataStore;
  54 import org.oran.pmproducer.filter.PmReportFilter;
  55 import org.oran.pmproducer.r1.ConsumerJobInfo;
  56 import org.oran.pmproducer.repository.InfoType;
  57 import org.oran.pmproducer.repository.InfoTypes;
  58 import org.oran.pmproducer.repository.Job;
  59 import org.oran.pmproducer.repository.Job.Statistics;
  60 import org.oran.pmproducer.repository.Jobs;
  61 import org.oran.pmproducer.tasks.NewFileEvent;
  62 import org.oran.pmproducer.tasks.TopicListener;
  63 import org.oran.pmproducer.tasks.TopicListeners;
  64 import org.slf4j.Logger;
  65 import org.slf4j.LoggerFactory;
  66 import org.springframework.beans.factory.annotation.Autowired;
  67 import org.springframework.boot.test.context.SpringBootTest;
  68 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
  69 import org.springframework.boot.test.context.TestConfiguration;
  70 import org.springframework.boot.test.web.server.LocalServerPort;
  71 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
  72 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
  73 import org.springframework.context.annotation.Bean;
  74 import org.springframework.test.context.TestPropertySource;
  75
  76 import reactor.core.publisher.Flux;
  77 import reactor.kafka.sender.KafkaSender;
  78 import reactor.kafka.sender.SenderOptions;
  79 import reactor.kafka.sender.SenderRecord;
  80
  81 @SuppressWarnings("java:S3577") // Rename class
  82 @SpringBootTest(webEnvironment = WebEnvironment.DEFINED_PORT)
  83 @TestPropertySource(properties = { //
  84         "server.ssl.key-store=./config/keystore.jks", //
  85         "app.webclient.trust-store=./config/truststore.jks", //
  86         "app.configuration-filepath=./src/test/resources/test_application_configuration.json", //
  87         "app.pm-files-path=./src/test/resources/", //
  88         "app.s3.locksBucket=ropfilelocks", //
  89         "app.pm-files-path=/tmp/dmaapadaptor", //
  90         "app.s3.bucket=dmaaptest" //
  91 }) //
  92 class IntegrationWithKafka {
  93
  94     final static String PM_TYPE_ID = "PmDataOverKafka";
  95
  96     @Autowired
  97     private ApplicationConfig applicationConfig;
  98
  99     @Autowired
 100     private Jobs jobs;
 101
 102     @Autowired
 103     private InfoTypes types;
 104
 105     @Autowired
 106     private IcsSimulatorController icsSimulatorController;
 107
 108     @Autowired
 109     private TopicListeners topicListeners;
 110
 111     @Autowired
 112     private SecurityContext securityContext;
 113
 114     private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create();
 115
 116     private final Logger logger = LoggerFactory.getLogger(IntegrationWithKafka.class);
 117
 118     @LocalServerPort
 119     int localServerHttpPort;
 120
 121     static class TestApplicationConfig extends ApplicationConfig {
 122         @Override
 123         public String getIcsBaseUrl() {
 124             return thisProcessUrl();
 125         }
 126
 127         @Override
 128         public String getDmaapBaseUrl() {
 129             return thisProcessUrl();
 130         }
 131
 132         @Override
 133         public String getSelfUrl() {
 134             return thisProcessUrl();
 135         }
 136
 137         private String thisProcessUrl() {
 138             final String url = "https://localhost:" + getLocalServerHttpPort();
 139             return url;
 140         }
 141     }
 142
 143     /**
 144      * Overrides the BeanFactory.
 145      */
 146     @TestConfiguration
 147     static class TestBeanFactory extends BeanFactory {
 148
 149         @Override
 150         @Bean
 151         public ServletWebServerFactory servletContainer() {
 152             return new TomcatServletWebServerFactory();
 153         }
 154
 155         @Override
 156         @Bean
 157         public ApplicationConfig getApplicationConfig() {
 158             TestApplicationConfig cfg = new TestApplicationConfig();
 159             return cfg;
 160         }
 161     }
 162
 163     private static class KafkaReceiver {
 164         public final String OUTPUT_TOPIC;
 165         private TopicListener.DataFromTopic receivedKafkaOutput;
 166         private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 167         private final ApplicationConfig applicationConfig;
 168
 169         int count = 0;
 170
 171         public KafkaReceiver(ApplicationConfig applicationConfig, String outputTopic, SecurityContext securityContext,
 172                 String groupId) {
 173             this.applicationConfig = applicationConfig;
 174             this.OUTPUT_TOPIC = outputTopic;
 175
 176             // Create a listener to the output topic. The TopicListener happens to be
 177             // suitable for that,
 178             InfoType type = InfoType.builder() //
 179                     .id("TestReceiver_" + outputTopic) //
 180                     .kafkaInputTopic(OUTPUT_TOPIC) //
 181                     .build();
 182
 183             TopicListener topicListener = new TopicListener(applicationConfig, type);
 184             if (groupId != null) {
 185                 topicListener.setKafkaGroupId(groupId);
 186             }
 187
 188             topicListener.getFlux() //
 189                     .map(this::unzip) //
 190                     .doOnNext(this::set) //
 191                     .doFinally(sig -> logger.info("Finally " + sig)) //
 192                     .subscribe();
 193         }
 194
 195         private TopicListener.DataFromTopic unzip(TopicListener.DataFromTopic receivedKafkaOutput) {
 196             if (this.applicationConfig.isZipOutput() != receivedKafkaOutput.isZipped()) {
 197                 logger.error("********* ERROR received zipped: {}, exp zipped: {}", receivedKafkaOutput.isZipped(),
 198                         this.applicationConfig.isZipOutput());
 199             }
 200
 201             if (!receivedKafkaOutput.isZipped()) {
 202                 return receivedKafkaOutput;
 203             }
 204             try {
 205                 byte[] unzipped = TopicListener.unzip(receivedKafkaOutput.value);
 206                 return new TopicListener.DataFromTopic("typeId", null, unzipped, receivedKafkaOutput.key);
 207             } catch (IOException e) {
 208                 logger.error("********* ERROR ", e.getMessage());
 209                 return null;
 210             }
 211         }
 212
 213         private void set(TopicListener.DataFromTopic receivedKafkaOutput) {
 214             this.receivedKafkaOutput = receivedKafkaOutput;
 215             this.count++;
 216             if (logger.isDebugEnabled()) {
 217                 logger.debug("*** received data on topic: {}", OUTPUT_TOPIC);
 218                 logger.debug("*** received typeId: {}", receivedKafkaOutput.getTypeIdFromHeaders());
 219             }
 220         }
 221
 222         void reset() {
 223             this.receivedKafkaOutput = new TopicListener.DataFromTopic("", null, null, null);
 224             this.count = 0;
 225         }
 226     }
 227
 228     private static KafkaReceiver kafkaReceiver;
 229     private static KafkaReceiver kafkaReceiver2;
 230
 231     @BeforeEach
 232     void init() {
 233         this.applicationConfig.setZipOutput(false);
 234
 235         if (kafkaReceiver == null) {
 236             kafkaReceiver = new KafkaReceiver(this.applicationConfig, "ouputTopic", this.securityContext, null);
 237             kafkaReceiver2 = new KafkaReceiver(this.applicationConfig, "ouputTopic2", this.securityContext, null);
 238         }
 239         kafkaReceiver.reset();
 240         kafkaReceiver2.reset();
 241
 242         DataStore fileStore = this.dataStore();
 243         fileStore.create(DataStore.Bucket.FILES).block();
 244         fileStore.create(DataStore.Bucket.LOCKS).block();
 245
 246     }
 247
 248     @AfterEach
 249     void reset() {
 250         for (Job job : this.jobs.getAll()) {
 251             this.icsSimulatorController.deleteJob(job.getId(), restClient());
 252         }
 253         await().untilAsserted(() -> assertThat(this.jobs.size()).isZero());
 254         await().untilAsserted(() -> assertThat(this.topicListeners.getDataDistributors().keySet()).isEmpty());
 255
 256         this.icsSimulatorController.testResults.reset();
 257
 258         DataStore fileStore = dataStore();
 259         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 260         fileStore.deleteBucket(DataStore.Bucket.LOCKS).block();
 261     }
 262
 263     private AsyncRestClient restClient(boolean useTrustValidation) {
 264         WebClientConfig config = this.applicationConfig.getWebClientConfig();
 265         HttpProxyConfig httpProxyConfig = HttpProxyConfig.builder() //
 266                 .httpProxyHost("") //
 267                 .httpProxyPort(0) //
 268                 .build();
 269         config = WebClientConfig.builder() //
 270                 .keyStoreType(config.getKeyStoreType()) //
 271                 .keyStorePassword(config.getKeyStorePassword()) //
 272                 .keyStore(config.getKeyStore()) //
 273                 .keyPassword(config.getKeyPassword()) //
 274                 .isTrustStoreUsed(useTrustValidation) //
 275                 .trustStore(config.getTrustStore()) //
 276                 .trustStorePassword(config.getTrustStorePassword()) //
 277                 .httpProxyConfig(httpProxyConfig).build();
 278
 279         AsyncRestClientFactory restClientFactory = new AsyncRestClientFactory(config, securityContext);
 280         return restClientFactory.createRestClientNoHttpProxy(baseUrl());
 281     }
 282
 283     private AsyncRestClient restClient() {
 284         return restClient(false);
 285     }
 286
 287     private String baseUrl() {
 288         return "https://localhost:" + this.applicationConfig.getLocalServerHttpPort();
 289     }
 290
 291     private static Object jsonObject(String json) {
 292         try {
 293             return JsonParser.parseString(json).getAsJsonObject();
 294         } catch (Exception e) {
 295             throw new NullPointerException(e.toString());
 296         }
 297     }
 298
 299     public static ConsumerJobInfo consumerJobInfoKafka(String kafkaBootstrapServers, String topic,
 300             PmReportFilter.FilterData filterData) {
 301         try {
 302             Job.Parameters.KafkaDeliveryInfo deliveryInfo = Job.Parameters.KafkaDeliveryInfo.builder() //
 303                     .topic(topic) //
 304                     .bootStrapServers(kafkaBootstrapServers) //
 305                     .build();
 306             Job.Parameters param = Job.Parameters.builder() //
 307                     .filter(filterData) //
 308                     .deliveryInfo(deliveryInfo) //
 309                     .build();
 310
 311             String str = gson.toJson(param);
 312             Object parametersObj = jsonObject(str);
 313
 314             return new ConsumerJobInfo(PM_TYPE_ID, parametersObj, "owner", "");
 315         } catch (Exception e) {
 316             return null;
 317         }
 318     }
 319
 320     private SenderOptions<byte[], byte[]> kafkaSenderOptions() {
 321         String bootstrapServers = this.applicationConfig.getKafkaBootStrapServers();
 322
 323         Map<String, Object> props = new HashMap<>();
 324         props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
 325         // props.put(ProducerConfig.CLIENT_ID_CONFIG, "sample-producerx");
 326         props.put(ProducerConfig.ACKS_CONFIG, "all");
 327         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
 328         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
 329         return SenderOptions.create(props);
 330     }
 331
 332     private SenderRecord<byte[], byte[], Integer> kafkaSenderRecord(String data, String key, String typeId) {
 333         final InfoType infoType = this.types.get(typeId);
 334         int correlationMetadata = 2;
 335         return SenderRecord.create(new ProducerRecord<>(infoType.getKafkaInputTopic(), key.getBytes(), data.getBytes()),
 336                 correlationMetadata);
 337     }
 338
 339     private void sendDataToKafka(Flux<SenderRecord<byte[], byte[], Integer>> dataToSend) {
 340         final KafkaSender<byte[], byte[]> sender = KafkaSender.create(kafkaSenderOptions());
 341
 342         sender.send(dataToSend) //
 343                 .doOnError(e -> logger.error("Send failed", e)) //
 344                 .blockLast();
 345
 346         sender.close();
 347     }
 348
 349     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 350     private static void waitForKafkaListener() throws InterruptedException {
 351         Thread.sleep(4000);
 352     }
 353
 354     private StatisticsCollection getStatistics() {
 355         String targetUri = baseUrl() + ProducerCallbacksController.STATISTICS_URL;
 356         String statsResp = restClient().get(targetUri).block();
 357         StatisticsCollection stats = gson.fromJson(statsResp, StatisticsCollection.class);
 358         return stats;
 359     }
 360
 361     @Builder
 362     static class CharacteristicsResult {
 363         long noOfFilesPerSecond;
 364         long noOfSentBytes;
 365         long noOfSentGigaBytes;
 366         long noOfSentObjects;
 367         long inputFileSize;
 368         long noOfReceivedFiles;
 369         long noOfReceivedBytes;
 370         long noOfSubscribers;
 371         long sizeOfSentObj;
 372         boolean zipOutput;
 373     }
 374
 375     private CharacteristicsResult getCharacteristicsResult(Instant startTime) {
 376         final long durationMs = Instant.now().toEpochMilli() - startTime.toEpochMilli();
 377         StatisticsCollection stats = getStatistics();
 378         long noOfSentBytes = 0;
 379         long noOfSentObjs = 0;
 380         for (Statistics s : stats.jobStatistics) {
 381             noOfSentBytes += s.getNoOfSentBytes();
 382             noOfSentObjs += s.getNoOfSentObjects();
 383         }
 384
 385         Statistics oneJobsStats = stats.jobStatistics.iterator().next();
 386
 387         return CharacteristicsResult.builder() //
 388                 .noOfSentBytes(noOfSentBytes) //
 389                 .noOfSentObjects(noOfSentObjs) //
 390                 .noOfSentGigaBytes(noOfSentBytes / (1024 * 1024)) //
 391                 .noOfSubscribers(stats.jobStatistics.size()) //
 392                 .zipOutput(this.applicationConfig.isZipOutput()) //
 393                 .noOfFilesPerSecond((oneJobsStats.getNoOfReceivedObjects() * 1000) / durationMs) //
 394                 .noOfReceivedBytes(oneJobsStats.getNoOfReceivedBytes()) //
 395                 .inputFileSize(oneJobsStats.getNoOfReceivedBytes() / oneJobsStats.getNoOfReceivedObjects()) //
 396                 .noOfReceivedFiles(oneJobsStats.getNoOfReceivedObjects()) //
 397                 .sizeOfSentObj(oneJobsStats.getNoOfSentBytes() / oneJobsStats.getNoOfSentObjects()) //
 398                 .build();
 399     }
 400
 401     private void printCharacteristicsResult(String str, Instant startTime, int noOfIterations) {
 402         final long durationMs = Instant.now().toEpochMilli() - startTime.toEpochMilli();
 403         logger.info("*** {} Duration ({} ms),  objects/second: {}", str, durationMs,
 404                 (noOfIterations * 1000) / durationMs);
 405
 406         System.out.println("--------------");
 407         System.out.println(gson.toJson(getCharacteristicsResult(startTime)));
 408         System.out.println("--------------");
 409
 410     }
 411
 412     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 413     @Test
 414     void kafkaCharacteristics_pmFilter_s3() throws Exception {
 415         // Filter PM reports and sent to two jobs over Kafka
 416
 417         final String JOB_ID = "kafkaCharacteristics";
 418         final String JOB_ID2 = "kafkaCharacteristics2";
 419
 420         // Register producer, Register types
 421         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 422         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 423
 424         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 425
 426         filterData.addMeasTypes("NRCellCU", "pmCounterNumber0");
 427
 428         this.icsSimulatorController.addJob(consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(),
 429                 kafkaReceiver.OUTPUT_TOPIC, filterData), JOB_ID, restClient());
 430         this.icsSimulatorController.addJob(consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(),
 431                 kafkaReceiver2.OUTPUT_TOPIC, filterData), JOB_ID2, restClient());
 432
 433         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(2));
 434         waitForKafkaListener();
 435
 436         final int NO_OF_OBJECTS = 10;
 437
 438         Instant startTime = Instant.now();
 439
 440         final String FILE_NAME = "A20000626.2315+0200-2330+0200_HTTPS-6-73.json";
 441
 442         DataStore fileStore = dataStore();
 443
 444         fileStore.create(DataStore.Bucket.FILES).block();
 445         fileStore.copyFileTo(Path.of("./src/test/resources/" + FILE_NAME), FILE_NAME).block();
 446
 447         String eventAsString = newFileEvent(FILE_NAME);
 448         var dataToSend = Flux.range(1, NO_OF_OBJECTS).map(i -> kafkaSenderRecord(eventAsString, "key", PM_TYPE_ID));
 449         sendDataToKafka(dataToSend);
 450
 451         while (kafkaReceiver.count < NO_OF_OBJECTS) {
 452             logger.info("sleeping {}", kafkaReceiver.count);
 453             Thread.sleep(1000 * 1);
 454         }
 455
 456         String msgString = kafkaReceiver.receivedKafkaOutput.valueAsString();
 457         assertThat(msgString).contains("pmCounterNumber0");
 458         assertThat(msgString).doesNotContain("pmCounterNumber1");
 459
 460         printCharacteristicsResult("kafkaCharacteristics_pmFilter_s3", startTime, NO_OF_OBJECTS);
 461         logger.info("***  kafkaReceiver2 :" + kafkaReceiver.count);
 462
 463         assertThat(kafkaReceiver.count).isEqualTo(NO_OF_OBJECTS);
 464     }
 465
 466     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 467     @Test
 468     void kafkaCharacteristics_manyPmJobs() throws Exception {
 469         // Filter PM reports and sent to many jobs over Kafka
 470
 471         this.applicationConfig.setZipOutput(false);
 472
 473         // Register producer, Register types
 474         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 475         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 476
 477         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 478         final int NO_OF_COUNTERS = 2;
 479         for (int i = 0; i < NO_OF_COUNTERS; ++i) {
 480             filterData.addMeasTypes("NRCellCU", "pmCounterNumber" + i);
 481         }
 482
 483         final int NO_OF_JOBS = 150;
 484
 485         ArrayList<KafkaReceiver> receivers = new ArrayList<>();
 486         for (int i = 0; i < NO_OF_JOBS; ++i) {
 487             final String outputTopic = "manyJobs_" + i;
 488             this.icsSimulatorController.addJob(
 489                     consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(), outputTopic, filterData),
 490                     outputTopic, restClient());
 491             KafkaReceiver receiver = new KafkaReceiver(this.applicationConfig, outputTopic, this.securityContext, null);
 492             receivers.add(receiver);
 493         }
 494
 495         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(NO_OF_JOBS));
 496         waitForKafkaListener();
 497
 498         final int NO_OF_OBJECTS = 1000;
 499
 500         Instant startTime = Instant.now();
 501
 502         final String FILE_NAME = "A20000626.2315+0200-2330+0200_HTTPS-6-73.json.gz";
 503
 504         DataStore fileStore = dataStore();
 505
 506         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 507         fileStore.create(DataStore.Bucket.FILES).block();
 508         fileStore.copyFileTo(Path.of("./src/test/resources/" + FILE_NAME), FILE_NAME).block();
 509
 510         String eventAsString = newFileEvent(FILE_NAME);
 511         var dataToSend = Flux.range(1, NO_OF_OBJECTS).map(i -> kafkaSenderRecord(eventAsString, "key", PM_TYPE_ID));
 512         sendDataToKafka(dataToSend);
 513
 514         logger.info("sleeping {}", kafkaReceiver.count);
 515         while (receivers.get(0).count < NO_OF_OBJECTS) {
 516             if (kafkaReceiver.count > 0) {
 517                 logger.info("sleeping {}", receivers.get(0).count);
 518             }
 519
 520             Thread.sleep(1000 * 1);
 521         }
 522
 523         printCharacteristicsResult("kafkaCharacteristics_manyPmJobs", startTime, NO_OF_OBJECTS);
 524
 525         for (KafkaReceiver receiver : receivers) {
 526             if (receiver.count != NO_OF_OBJECTS) {
 527                 System.out.println("** Unexpected no of jobs: " + receiver.OUTPUT_TOPIC + " " + receiver.count);
 528             }
 529         }
 530     }
 531
 532     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 533     @Test
 534     void kafkaCharacteristics_manyPmJobs_sharedTopic() throws Exception {
 535         // Filter PM reports and sent to many jobs over Kafka
 536
 537         this.applicationConfig.setZipOutput(false);
 538
 539         // Register producer, Register types
 540         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 541         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 542
 543         final int NO_OF_JOBS = 150;
 544         ArrayList<KafkaReceiver> receivers = new ArrayList<>();
 545         for (int i = 0; i < NO_OF_JOBS; ++i) {
 546             final String outputTopic = "kafkaCharacteristics_onePmJobs_manyReceivers";
 547             final String jobId = "manyJobs_" + i;
 548             PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 549             filterData.addMeasTypes("NRCellCU", "pmCounterNumber" + i); // all counters will be added
 550
 551             this.icsSimulatorController.addJob(
 552                     consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(), outputTopic, filterData),
 553                     jobId, restClient());
 554
 555             KafkaReceiver receiver =
 556                     new KafkaReceiver(this.applicationConfig, outputTopic, this.securityContext, "group_" + i);
 557             receivers.add(receiver);
 558         }
 559
 560         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(NO_OF_JOBS));
 561         waitForKafkaListener();
 562
 563         final int NO_OF_OBJECTS = 1000;
 564
 565         Instant startTime = Instant.now();
 566
 567         final String FILE_NAME = "A20000626.2315+0200-2330+0200_HTTPS-6-73.json.gz";
 568
 569         DataStore fileStore = dataStore();
 570
 571         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 572         fileStore.create(DataStore.Bucket.FILES).block();
 573         fileStore.copyFileTo(Path.of("./src/test/resources/" + FILE_NAME), FILE_NAME).block();
 574
 575         String eventAsString = newFileEvent(FILE_NAME);
 576         var dataToSend = Flux.range(1, NO_OF_OBJECTS).map(i -> kafkaSenderRecord(eventAsString, "key", PM_TYPE_ID));
 577         sendDataToKafka(dataToSend);
 578
 579         logger.info("sleeping {}", kafkaReceiver.count);
 580         for (KafkaReceiver receiver : receivers) {
 581             while (receiver.count < NO_OF_OBJECTS) {
 582                 if (kafkaReceiver.count > 0) {
 583                     logger.info("sleeping {}", receiver.count);
 584                 }
 585
 586                 Thread.sleep(1000 * 1);
 587             }
 588         }
 589
 590         printCharacteristicsResult("kafkaCharacteristics_manyPmJobs", startTime, NO_OF_OBJECTS);
 591
 592         for (KafkaReceiver receiver : receivers) {
 593             if (receiver.count != NO_OF_OBJECTS) {
 594                 System.out.println("** Unexpected no of objects: " + receiver.OUTPUT_TOPIC + " " + receiver.count);
 595             }
 596         }
 597
 598         Thread.sleep(1000 * 5);
 599     }
 600
 601     private String newFileEvent(String fileName) {
 602         NewFileEvent event = NewFileEvent.builder() //
 603                 .filename(fileName) //
 604                 .build();
 605         return gson.toJson(event);
 606     }
 607
 608     private DataStore dataStore() {
 609         return DataStore.create(this.applicationConfig);
 610     }
 611
 612     @Test
 613     void testHistoricalData() throws Exception {
 614         // test
 615         final String JOB_ID = "testHistoricalData";
 616
 617         DataStore fileStore = dataStore();
 618
 619         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 620         fileStore.create(DataStore.Bucket.FILES).block();
 621         fileStore.create(DataStore.Bucket.LOCKS).block();
 622
 623         fileStore.copyFileTo(Path.of("./src/test/resources/pm_report.json"),
 624                 "O-DU-1122/A20000626.2315+0200-2330+0200_HTTPS-6-73.json").block();
 625
 626         fileStore.copyFileTo(Path.of("./src/test/resources/pm_report.json"), "OTHER_SOURCENAME/test.json").block();
 627
 628         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 629
 630         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 631         filterData.getSourceNames().add("O-DU-1122");
 632         filterData.setPmRopStartTime("1999-12-27T10:50:44.000-08:00");
 633
 634         filterData.setPmRopEndTime(OffsetDateTime.now().toString());
 635
 636         this.icsSimulatorController.addJob(consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(),
 637                 kafkaReceiver.OUTPUT_TOPIC, filterData), JOB_ID, restClient());
 638         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(1));
 639
 640         await().untilAsserted(() -> assertThat(kafkaReceiver.count).isPositive());
 641     }
 642
 643 }