pmproducer/src/test/java/org/oran/pmproducer/IntegrationWithKafka.java

   1 /*-
   2  * ========================LICENSE_START=================================
   3  * O-RAN-SC
   4  * %%
   5  * Copyright (C) 2023 Nordix Foundation
   6  * %%
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ========================LICENSE_END===================================
  19  */
  20
  21 package org.oran.pmproducer;
  22
  23 import static org.assertj.core.api.Assertions.assertThat;
  24 import static org.awaitility.Awaitility.await;
  25
  26 import com.google.gson.JsonParser;
  27
  28 import java.io.IOException;
  29 import java.lang.invoke.MethodHandles;
  30 import java.nio.file.Path;
  31 import java.time.Instant;
  32 import java.time.OffsetDateTime;
  33 import java.util.ArrayList;
  34 import java.util.HashMap;
  35 import java.util.Map;
  36
  37 import lombok.Builder;
  38
  39 import org.apache.kafka.clients.producer.ProducerConfig;
  40 import org.apache.kafka.clients.producer.ProducerRecord;
  41 import org.apache.kafka.common.serialization.ByteArraySerializer;
  42 import org.junit.jupiter.api.AfterEach;
  43 import org.junit.jupiter.api.BeforeEach;
  44 import org.junit.jupiter.api.Test;
  45 import org.oran.pmproducer.clients.AsyncRestClient;
  46 import org.oran.pmproducer.clients.AsyncRestClientFactory;
  47 import org.oran.pmproducer.configuration.ApplicationConfig;
  48 import org.oran.pmproducer.configuration.WebClientConfig;
  49 import org.oran.pmproducer.configuration.WebClientConfig.HttpProxyConfig;
  50 import org.oran.pmproducer.controllers.ProducerCallbacksController;
  51 import org.oran.pmproducer.controllers.ProducerCallbacksController.StatisticsCollection;
  52 import org.oran.pmproducer.datastore.DataStore;
  53 import org.oran.pmproducer.filter.PmReportFilter;
  54 import org.oran.pmproducer.oauth2.SecurityContext;
  55 import org.oran.pmproducer.r1.ConsumerJobInfo;
  56 import org.oran.pmproducer.repository.InfoType;
  57 import org.oran.pmproducer.repository.InfoTypes;
  58 import org.oran.pmproducer.repository.Job;
  59 import org.oran.pmproducer.repository.Job.Statistics;
  60 import org.oran.pmproducer.repository.Jobs;
  61 import org.oran.pmproducer.tasks.NewFileEvent;
  62 import org.oran.pmproducer.tasks.TopicListener;
  63 import org.oran.pmproducer.tasks.TopicListeners;
  64 import org.slf4j.Logger;
  65 import org.slf4j.LoggerFactory;
  66 import org.springframework.beans.factory.annotation.Autowired;
  67 import org.springframework.boot.test.context.SpringBootTest;
  68 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
  69 import org.springframework.boot.test.context.TestConfiguration;
  70 import org.springframework.boot.test.web.server.LocalServerPort;
  71 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
  72 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
  73 import org.springframework.context.annotation.Bean;
  74 import org.springframework.test.context.TestPropertySource;
  75
  76 import reactor.core.publisher.Flux;
  77 import reactor.kafka.sender.KafkaSender;
  78 import reactor.kafka.sender.SenderOptions;
  79 import reactor.kafka.sender.SenderRecord;
  80
  81 @SuppressWarnings("java:S3577") // Rename class
  82 @SpringBootTest(webEnvironment = WebEnvironment.DEFINED_PORT)
  83 @TestPropertySource(properties = { //
  84         "server.ssl.key-store=./config/keystore.jks", //
  85         "app.webclient.trust-store=./config/truststore.jks", //
  86         "app.configuration-filepath=./src/test/resources/test_application_configuration.json", //
  87         "app.pm-files-path=./src/test/resources/", //
  88         "app.s3.locksBucket=ropfilelocks", //
  89         "app.pm-files-path=/tmp/dmaapadaptor", //
  90         "app.s3.bucket=dmaaptest", //
  91         "app.auth-token-file=src/test/resources/jwtToken.b64", //
  92         "app.kafka.use-oath-token=false"}) //
  93 class IntegrationWithKafka {
  94
  95     final static String PM_TYPE_ID = "PmDataOverKafka";
  96
  97     @Autowired
  98     private ApplicationConfig applicationConfig;
  99
 100     @Autowired
 101     private Jobs jobs;
 102
 103     @Autowired
 104     private InfoTypes types;
 105
 106     @Autowired
 107     private IcsSimulatorController icsSimulatorController;
 108
 109     @Autowired
 110     private TopicListeners topicListeners;
 111
 112     @Autowired
 113     private SecurityContext securityContext;
 114
 115     private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create();
 116
 117     private final Logger logger = LoggerFactory.getLogger(IntegrationWithKafka.class);
 118
 119     @LocalServerPort
 120     int localServerHttpPort;
 121
 122     static class TestApplicationConfig extends ApplicationConfig {
 123         @Override
 124         public String getIcsBaseUrl() {
 125             return thisProcessUrl();
 126         }
 127
 128         @Override
 129         public String getSelfUrl() {
 130             return thisProcessUrl();
 131         }
 132
 133         private String thisProcessUrl() {
 134             final String url = "https://localhost:" + getLocalServerHttpPort();
 135             return url;
 136         }
 137     }
 138
 139     /**
 140      * Overrides the BeanFactory.
 141      */
 142     @TestConfiguration
 143     static class TestBeanFactory extends BeanFactory {
 144
 145         @Override
 146         @Bean
 147         public ServletWebServerFactory servletContainer() {
 148             return new TomcatServletWebServerFactory();
 149         }
 150
 151         @Override
 152         @Bean
 153         public ApplicationConfig getApplicationConfig() {
 154             TestApplicationConfig cfg = new TestApplicationConfig();
 155             return cfg;
 156         }
 157     }
 158
 159     private static class KafkaReceiver {
 160         public final String OUTPUT_TOPIC;
 161         private TopicListener.DataFromTopic receivedKafkaOutput;
 162         private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 163         private final ApplicationConfig applicationConfig;
 164
 165         int count = 0;
 166
 167         public KafkaReceiver(ApplicationConfig applicationConfig, String outputTopic, SecurityContext securityContext,
 168                 String groupId) {
 169             this.applicationConfig = applicationConfig;
 170             this.OUTPUT_TOPIC = outputTopic;
 171
 172             // Create a listener to the output topic. The TopicListener happens to be
 173             // suitable for that,
 174             InfoType type = InfoType.builder() //
 175                     .id("TestReceiver_" + outputTopic) //
 176                     .kafkaInputTopic(OUTPUT_TOPIC) //
 177                     .build();
 178
 179             TopicListener topicListener = new TopicListener(applicationConfig, type);
 180             if (groupId != null) {
 181                 topicListener.setKafkaGroupId(groupId);
 182             }
 183
 184             topicListener.getFlux() //
 185                     .map(this::unzip) //
 186                     .doOnNext(this::set) //
 187                     .doFinally(sig -> logger.info("Finally " + sig)) //
 188                     .subscribe();
 189         }
 190
 191         private TopicListener.DataFromTopic unzip(TopicListener.DataFromTopic receivedKafkaOutput) {
 192             if (this.applicationConfig.isZipOutput() != receivedKafkaOutput.isZipped()) {
 193                 logger.error("********* ERROR received zipped: {}, exp zipped: {}", receivedKafkaOutput.isZipped(),
 194                         this.applicationConfig.isZipOutput());
 195             }
 196
 197             if (!receivedKafkaOutput.isZipped()) {
 198                 return receivedKafkaOutput;
 199             }
 200             try {
 201                 byte[] unzipped = TopicListener.unzip(receivedKafkaOutput.value);
 202                 return new TopicListener.DataFromTopic("typeId", null, unzipped, receivedKafkaOutput.key);
 203             } catch (IOException e) {
 204                 logger.error("********* ERROR ", e.getMessage());
 205                 return null;
 206             }
 207         }
 208
 209         private void set(TopicListener.DataFromTopic receivedKafkaOutput) {
 210             this.receivedKafkaOutput = receivedKafkaOutput;
 211             this.count++;
 212             if (logger.isDebugEnabled()) {
 213                 logger.debug("*** received data on topic: {}", OUTPUT_TOPIC);
 214                 logger.debug("*** received typeId: {}", receivedKafkaOutput.getTypeIdFromHeaders());
 215             }
 216         }
 217
 218         void reset() {
 219             this.receivedKafkaOutput = new TopicListener.DataFromTopic("", null, null, null);
 220             this.count = 0;
 221         }
 222     }
 223
 224     private static KafkaReceiver kafkaReceiver;
 225     private static KafkaReceiver kafkaReceiver2;
 226
 227     @BeforeEach
 228     void init() {
 229         this.applicationConfig.setZipOutput(false);
 230
 231         if (kafkaReceiver == null) {
 232             kafkaReceiver = new KafkaReceiver(this.applicationConfig, "ouputTopic", this.securityContext, null);
 233             kafkaReceiver2 = new KafkaReceiver(this.applicationConfig, "ouputTopic2", this.securityContext, null);
 234         }
 235         kafkaReceiver.reset();
 236         kafkaReceiver2.reset();
 237
 238         DataStore fileStore = this.dataStore();
 239         fileStore.create(DataStore.Bucket.FILES).block();
 240         fileStore.create(DataStore.Bucket.LOCKS).block();
 241
 242     }
 243
 244     @AfterEach
 245     void reset() {
 246         for (Job job : this.jobs.getAll()) {
 247             this.icsSimulatorController.deleteJob(job.getId(), restClient());
 248         }
 249         await().untilAsserted(() -> assertThat(this.jobs.size()).isZero());
 250         await().untilAsserted(() -> assertThat(this.topicListeners.getDataDistributors().keySet()).isEmpty());
 251
 252         this.icsSimulatorController.testResults.reset();
 253
 254         DataStore fileStore = dataStore();
 255         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 256         fileStore.deleteBucket(DataStore.Bucket.LOCKS).block();
 257     }
 258
 259     private AsyncRestClient restClient(boolean useTrustValidation) {
 260         WebClientConfig config = this.applicationConfig.getWebClientConfig();
 261         HttpProxyConfig httpProxyConfig = HttpProxyConfig.builder() //
 262                 .httpProxyHost("") //
 263                 .httpProxyPort(0) //
 264                 .build();
 265         config = WebClientConfig.builder() //
 266                 .keyStoreType(config.getKeyStoreType()) //
 267                 .keyStorePassword(config.getKeyStorePassword()) //
 268                 .keyStore(config.getKeyStore()) //
 269                 .keyPassword(config.getKeyPassword()) //
 270                 .isTrustStoreUsed(useTrustValidation) //
 271                 .trustStore(config.getTrustStore()) //
 272                 .trustStorePassword(config.getTrustStorePassword()) //
 273                 .httpProxyConfig(httpProxyConfig).build();
 274
 275         AsyncRestClientFactory restClientFactory = new AsyncRestClientFactory(config, securityContext);
 276         return restClientFactory.createRestClientNoHttpProxy(baseUrl());
 277     }
 278
 279     private AsyncRestClient restClient() {
 280         return restClient(false);
 281     }
 282
 283     private String baseUrl() {
 284         return "https://localhost:" + this.applicationConfig.getLocalServerHttpPort();
 285     }
 286
 287     private static Object jsonObject(String json) {
 288         try {
 289             return JsonParser.parseString(json).getAsJsonObject();
 290         } catch (Exception e) {
 291             throw new NullPointerException(e.toString());
 292         }
 293     }
 294
 295     public static ConsumerJobInfo consumerJobInfoKafka(String kafkaBootstrapServers, String topic,
 296             PmReportFilter.FilterData filterData) {
 297         try {
 298             Job.Parameters.KafkaDeliveryInfo deliveryInfo = Job.Parameters.KafkaDeliveryInfo.builder() //
 299                     .topic(topic) //
 300                     .bootStrapServers(kafkaBootstrapServers) //
 301                     .build();
 302             Job.Parameters param = Job.Parameters.builder() //
 303                     .filter(filterData) //
 304                     .deliveryInfo(deliveryInfo) //
 305                     .build();
 306
 307             String str = gson.toJson(param);
 308             Object parametersObj = jsonObject(str);
 309
 310             return new ConsumerJobInfo(PM_TYPE_ID, parametersObj, "owner", "");
 311         } catch (Exception e) {
 312             return null;
 313         }
 314     }
 315
 316     private SenderOptions<byte[], byte[]> kafkaSenderOptions() {
 317         String bootstrapServers = this.applicationConfig.getKafkaBootStrapServers();
 318
 319         Map<String, Object> props = new HashMap<>();
 320         props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
 321         // props.put(ProducerConfig.CLIENT_ID_CONFIG, "sample-producerx");
 322         props.put(ProducerConfig.ACKS_CONFIG, "all");
 323         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
 324         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
 325
 326         // Security
 327         this.applicationConfig.addKafkaSecurityProps(props);
 328
 329         return SenderOptions.create(props);
 330     }
 331
 332     private SenderRecord<byte[], byte[], Integer> kafkaSenderRecord(String data, String key, String typeId) {
 333         final InfoType infoType = this.types.get(typeId);
 334         int correlationMetadata = 2;
 335         return SenderRecord.create(new ProducerRecord<>(infoType.getKafkaInputTopic(), key.getBytes(), data.getBytes()),
 336                 correlationMetadata);
 337     }
 338
 339     private void sendDataToKafka(Flux<SenderRecord<byte[], byte[], Integer>> dataToSend) {
 340         final KafkaSender<byte[], byte[]> sender = KafkaSender.create(kafkaSenderOptions());
 341
 342         sender.send(dataToSend) //
 343                 .doOnError(e -> logger.error("Send failed", e)) //
 344                 .blockLast();
 345
 346         sender.close();
 347     }
 348
 349     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 350     private static void waitForKafkaListener() throws InterruptedException {
 351         Thread.sleep(4000);
 352     }
 353
 354     private StatisticsCollection getStatistics() {
 355         String targetUri = baseUrl() + ProducerCallbacksController.STATISTICS_URL;
 356         String statsResp = restClient().get(targetUri).block();
 357         StatisticsCollection stats = gson.fromJson(statsResp, StatisticsCollection.class);
 358         return stats;
 359     }
 360
 361     @Builder
 362     static class CharacteristicsResult {
 363         long noOfFilesPerSecond;
 364         long noOfSentBytes;
 365         long noOfSentGigaBytes;
 366         long noOfSentObjects;
 367         long inputFileSize;
 368         long noOfReceivedFiles;
 369         long noOfReceivedBytes;
 370         long noOfSubscribers;
 371         long sizeOfSentObj;
 372         boolean zipOutput;
 373     }
 374
 375     private CharacteristicsResult getCharacteristicsResult(Instant startTime) {
 376         final long durationMs = Instant.now().toEpochMilli() - startTime.toEpochMilli();
 377         StatisticsCollection stats = getStatistics();
 378         long noOfSentBytes = 0;
 379         long noOfSentObjs = 0;
 380         for (Statistics s : stats.jobStatistics) {
 381             noOfSentBytes += s.getNoOfSentBytes();
 382             noOfSentObjs += s.getNoOfSentObjects();
 383         }
 384
 385         Statistics oneJobsStats = stats.jobStatistics.iterator().next();
 386
 387         return CharacteristicsResult.builder() //
 388                 .noOfSentBytes(noOfSentBytes) //
 389                 .noOfSentObjects(noOfSentObjs) //
 390                 .noOfSentGigaBytes(noOfSentBytes / (1024 * 1024)) //
 391                 .noOfSubscribers(stats.jobStatistics.size()) //
 392                 .zipOutput(this.applicationConfig.isZipOutput()) //
 393                 .noOfFilesPerSecond((oneJobsStats.getNoOfReceivedObjects() * 1000) / durationMs) //
 394                 .noOfReceivedBytes(oneJobsStats.getNoOfReceivedBytes()) //
 395                 .inputFileSize(oneJobsStats.getNoOfReceivedBytes() / oneJobsStats.getNoOfReceivedObjects()) //
 396                 .noOfReceivedFiles(oneJobsStats.getNoOfReceivedObjects()) //
 397                 .sizeOfSentObj(oneJobsStats.getNoOfSentBytes() / oneJobsStats.getNoOfSentObjects()) //
 398                 .build();
 399     }
 400
 401     private void printCharacteristicsResult(String str, Instant startTime, int noOfIterations) {
 402         final long durationMs = Instant.now().toEpochMilli() - startTime.toEpochMilli();
 403         logger.info("*** {} Duration ({} ms),  objects/second: {}", str, durationMs,
 404                 (noOfIterations * 1000) / durationMs);
 405
 406         System.out.println("--------------");
 407         System.out.println(gson.toJson(getCharacteristicsResult(startTime)));
 408         System.out.println("--------------");
 409
 410     }
 411
 412     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 413     @Test
 414     void kafkaCharacteristics_pmFilter_s3() throws Exception {
 415         // Filter PM reports and sent to two jobs over Kafka
 416
 417         final String JOB_ID = "kafkaCharacteristics";
 418         final String JOB_ID2 = "kafkaCharacteristics2";
 419
 420         // Register producer, Register types
 421         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 422         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 423
 424         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 425
 426         filterData.addMeasTypes("NRCellCU", "pmCounterNumber0");
 427
 428         this.icsSimulatorController.addJob(consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(),
 429                 kafkaReceiver.OUTPUT_TOPIC, filterData), JOB_ID, restClient());
 430         this.icsSimulatorController.addJob(consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(),
 431                 kafkaReceiver2.OUTPUT_TOPIC, filterData), JOB_ID2, restClient());
 432
 433         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(2));
 434         waitForKafkaListener();
 435
 436         final int NO_OF_OBJECTS = 10;
 437
 438         Instant startTime = Instant.now();
 439
 440         final String FILE_NAME = "A20000626.2315+0200-2330+0200_HTTPS-6-73.json";
 441
 442         DataStore fileStore = dataStore();
 443
 444         fileStore.create(DataStore.Bucket.FILES).block();
 445         fileStore.copyFileTo(Path.of("./src/test/resources/" + FILE_NAME), FILE_NAME).block();
 446
 447         String eventAsString = newFileEvent(FILE_NAME);
 448         var dataToSend = Flux.range(1, NO_OF_OBJECTS).map(i -> kafkaSenderRecord(eventAsString, "key", PM_TYPE_ID));
 449         sendDataToKafka(dataToSend);
 450
 451         while (kafkaReceiver.count < NO_OF_OBJECTS) {
 452             logger.info("sleeping {}", kafkaReceiver.count);
 453             Thread.sleep(1000 * 1);
 454         }
 455
 456         String msgString = kafkaReceiver.receivedKafkaOutput.valueAsString();
 457         assertThat(msgString).contains("pmCounterNumber0");
 458         assertThat(msgString).doesNotContain("pmCounterNumber1");
 459
 460         printCharacteristicsResult("kafkaCharacteristics_pmFilter_s3", startTime, NO_OF_OBJECTS);
 461         logger.info("***  kafkaReceiver2 :" + kafkaReceiver.count);
 462
 463         assertThat(kafkaReceiver.count).isEqualTo(NO_OF_OBJECTS);
 464     }
 465
 466     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 467     @Test
 468     void kafkaCharacteristics_manyPmJobs() throws Exception {
 469         // Filter PM reports and sent to many jobs over Kafka
 470
 471         this.applicationConfig.setZipOutput(false);
 472
 473         // Register producer, Register types
 474         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 475         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 476
 477         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 478         final int NO_OF_COUNTERS = 2;
 479         for (int i = 0; i < NO_OF_COUNTERS; ++i) {
 480             filterData.addMeasTypes("NRCellCU", "pmCounterNumber" + i);
 481         }
 482
 483         final int NO_OF_JOBS = 150;
 484
 485         ArrayList<KafkaReceiver> receivers = new ArrayList<>();
 486         for (int i = 0; i < NO_OF_JOBS; ++i) {
 487             final String outputTopic = "manyJobs_" + i;
 488             this.icsSimulatorController.addJob(
 489                     consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(), outputTopic, filterData),
 490                     outputTopic, restClient());
 491             KafkaReceiver receiver = new KafkaReceiver(this.applicationConfig, outputTopic, this.securityContext, null);
 492             receivers.add(receiver);
 493         }
 494
 495         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(NO_OF_JOBS));
 496         waitForKafkaListener();
 497
 498         final int NO_OF_OBJECTS = 1000;
 499
 500         Instant startTime = Instant.now();
 501
 502         final String FILE_NAME = "A20000626.2315+0200-2330+0200_HTTPS-6-73.json.gz";
 503
 504         DataStore fileStore = dataStore();
 505
 506         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 507         fileStore.create(DataStore.Bucket.FILES).block();
 508         fileStore.copyFileTo(Path.of("./src/test/resources/" + FILE_NAME), FILE_NAME).block();
 509
 510         String eventAsString = newFileEvent(FILE_NAME);
 511         var dataToSend = Flux.range(1, NO_OF_OBJECTS).map(i -> kafkaSenderRecord(eventAsString, "key", PM_TYPE_ID));
 512         sendDataToKafka(dataToSend);
 513
 514         logger.info("sleeping {}", kafkaReceiver.count);
 515         while (receivers.get(0).count < NO_OF_OBJECTS) {
 516             if (kafkaReceiver.count > 0) {
 517                 logger.info("sleeping {}", receivers.get(0).count);
 518             }
 519
 520             Thread.sleep(1000 * 1);
 521         }
 522
 523         printCharacteristicsResult("kafkaCharacteristics_manyPmJobs", startTime, NO_OF_OBJECTS);
 524
 525         for (KafkaReceiver receiver : receivers) {
 526             if (receiver.count != NO_OF_OBJECTS) {
 527                 System.out.println("** Unexpected no of jobs: " + receiver.OUTPUT_TOPIC + " " + receiver.count);
 528             }
 529         }
 530     }
 531
 532     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 533     @Test
 534     void kafkaCharacteristics_manyPmJobs_sharedTopic() throws Exception {
 535         // Filter PM reports and sent to many jobs over Kafka
 536
 537         this.applicationConfig.setZipOutput(false);
 538
 539         // Register producer, Register types
 540         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 541         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 542
 543         final int NO_OF_JOBS = 150;
 544         ArrayList<KafkaReceiver> receivers = new ArrayList<>();
 545         for (int i = 0; i < NO_OF_JOBS; ++i) {
 546             final String outputTopic = "kafkaCharacteristics_onePmJobs_manyReceivers";
 547             final String jobId = "manyJobs_" + i;
 548             PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 549             filterData.addMeasTypes("NRCellCU", "pmCounterNumber" + i); // all counters will be added
 550
 551             this.icsSimulatorController.addJob(
 552                     consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(), outputTopic, filterData),
 553                     jobId, restClient());
 554
 555             KafkaReceiver receiver =
 556                     new KafkaReceiver(this.applicationConfig, outputTopic, this.securityContext, "group_" + i);
 557             receivers.add(receiver);
 558         }
 559
 560         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(NO_OF_JOBS));
 561         waitForKafkaListener();
 562
 563         final int NO_OF_OBJECTS = 1000;
 564
 565         Instant startTime = Instant.now();
 566
 567         final String FILE_NAME = "A20000626.2315+0200-2330+0200_HTTPS-6-73.json.gz";
 568
 569         DataStore fileStore = dataStore();
 570
 571         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 572         fileStore.create(DataStore.Bucket.FILES).block();
 573         fileStore.copyFileTo(Path.of("./src/test/resources/" + FILE_NAME), FILE_NAME).block();
 574
 575         String eventAsString = newFileEvent(FILE_NAME);
 576         var dataToSend = Flux.range(1, NO_OF_OBJECTS).map(i -> kafkaSenderRecord(eventAsString, "key", PM_TYPE_ID));
 577         sendDataToKafka(dataToSend);
 578
 579         logger.info("sleeping {}", kafkaReceiver.count);
 580         for (KafkaReceiver receiver : receivers) {
 581             while (receiver.count < NO_OF_OBJECTS) {
 582                 if (kafkaReceiver.count > 0) {
 583                     logger.info("sleeping {}", receiver.count);
 584                 }
 585
 586                 Thread.sleep(1000 * 1);
 587             }
 588         }
 589
 590         printCharacteristicsResult("kafkaCharacteristics_manyPmJobs", startTime, NO_OF_OBJECTS);
 591
 592         for (KafkaReceiver receiver : receivers) {
 593             if (receiver.count != NO_OF_OBJECTS) {
 594                 System.out.println("** Unexpected no of objects: " + receiver.OUTPUT_TOPIC + " " + receiver.count);
 595             }
 596         }
 597
 598         Thread.sleep(1000 * 5);
 599     }
 600
 601     private String newFileEvent(String fileName) {
 602         NewFileEvent event = NewFileEvent.builder() //
 603                 .filename(fileName) //
 604                 .build();
 605         return gson.toJson(event);
 606     }
 607
 608     private DataStore dataStore() {
 609         return DataStore.create(this.applicationConfig);
 610     }
 611
 612     @Test
 613     void testHistoricalData() throws Exception {
 614         // test
 615         final String JOB_ID = "testHistoricalData";
 616
 617         DataStore fileStore = dataStore();
 618
 619         fileStore.deleteBucket(DataStore.Bucket.FILES).block();
 620         fileStore.create(DataStore.Bucket.FILES).block();
 621         fileStore.create(DataStore.Bucket.LOCKS).block();
 622
 623         fileStore.copyFileTo(Path.of("./src/test/resources/pm_report.json"),
 624                 "O-DU-1122/A20000626.2315+0200-2330+0200_HTTPS-6-73.json").block();
 625
 626         fileStore.copyFileTo(Path.of("./src/test/resources/pm_report.json"), "OTHER_SOURCENAME/test.json").block();
 627
 628         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 629
 630         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
 631         filterData.getSourceNames().add("O-DU-1122");
 632         filterData.setPmRopStartTime("1999-12-27T10:50:44.000-08:00");
 633
 634         filterData.setPmRopEndTime(OffsetDateTime.now().toString());
 635
 636         this.icsSimulatorController.addJob(consumerJobInfoKafka(this.applicationConfig.getKafkaBootStrapServers(),
 637                 kafkaReceiver.OUTPUT_TOPIC, filterData), JOB_ID, restClient());
 638         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(1));
 639
 640         await().untilAsserted(() -> assertThat(kafkaReceiver.count).isPositive());
 641     }
 642
 643 }