dmaap-adaptor-java/src/test/java/org/oran/dmaapadapter/IntegrationWithKafka.java

   1 /*-
   2  * ========================LICENSE_START=================================
   3  * O-RAN-SC
   4  * %%
   5  * Copyright (C) 2021 Nordix Foundation
   6  * %%
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ========================LICENSE_END===================================
  19  */
  20
  21 package org.oran.dmaapadapter;
  22
  23 import static org.assertj.core.api.Assertions.assertThat;
  24 import static org.awaitility.Awaitility.await;
  25 import static org.junit.jupiter.api.Assertions.assertTrue;
  26
  27 import com.google.gson.JsonParser;
  28
  29 import java.time.Duration;
  30 import java.util.HashMap;
  31 import java.util.Map;
  32
  33 import org.apache.kafka.clients.producer.ProducerConfig;
  34 import org.apache.kafka.clients.producer.ProducerRecord;
  35 import org.apache.kafka.common.serialization.IntegerSerializer;
  36 import org.apache.kafka.common.serialization.StringSerializer;
  37 import org.junit.jupiter.api.AfterEach;
  38 import org.junit.jupiter.api.Test;
  39 import org.junit.jupiter.api.extension.ExtendWith;
  40 import org.oran.dmaapadapter.clients.AsyncRestClient;
  41 import org.oran.dmaapadapter.clients.AsyncRestClientFactory;
  42 import org.oran.dmaapadapter.configuration.ApplicationConfig;
  43 import org.oran.dmaapadapter.configuration.ImmutableHttpProxyConfig;
  44 import org.oran.dmaapadapter.configuration.ImmutableWebClientConfig;
  45 import org.oran.dmaapadapter.configuration.WebClientConfig;
  46 import org.oran.dmaapadapter.configuration.WebClientConfig.HttpProxyConfig;
  47 import org.oran.dmaapadapter.r1.ConsumerJobInfo;
  48 import org.oran.dmaapadapter.repository.InfoType;
  49 import org.oran.dmaapadapter.repository.InfoTypes;
  50 import org.oran.dmaapadapter.repository.Job;
  51 import org.oran.dmaapadapter.repository.Jobs;
  52 import org.oran.dmaapadapter.tasks.KafkaJobDataConsumer;
  53 import org.oran.dmaapadapter.tasks.KafkaTopicConsumers;
  54 import org.slf4j.Logger;
  55 import org.slf4j.LoggerFactory;
  56 import org.springframework.beans.factory.annotation.Autowired;
  57 import org.springframework.boot.test.context.SpringBootTest;
  58 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
  59 import org.springframework.boot.test.context.TestConfiguration;
  60 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
  61 import org.springframework.boot.web.server.LocalServerPort;
  62 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
  63 import org.springframework.context.annotation.Bean;
  64 import org.springframework.test.context.TestPropertySource;
  65 import org.springframework.test.context.junit.jupiter.SpringExtension;
  66
  67 import reactor.core.publisher.Flux;
  68 import reactor.kafka.sender.KafkaSender;
  69 import reactor.kafka.sender.SenderOptions;
  70 import reactor.kafka.sender.SenderRecord;
  71
  72 @SuppressWarnings("java:S3577") // Rename class
  73 @ExtendWith(SpringExtension.class)
  74 @SpringBootTest(webEnvironment = WebEnvironment.DEFINED_PORT)
  75 @TestPropertySource(properties = { //
  76         "server.ssl.key-store=./config/keystore.jks", //
  77         "app.webclient.trust-store=./config/truststore.jks", //
  78         "app.configuration-filepath=./src/test/resources/test_application_configuration.json"//
  79 })
  80 class IntegrationWithKafka {
  81
  82     final String TYPE_ID = "KafkaInformationType";
  83
  84     @Autowired
  85     private ApplicationConfig applicationConfig;
  86
  87     @Autowired
  88     private Jobs jobs;
  89
  90     @Autowired
  91     private InfoTypes types;
  92
  93     @Autowired
  94     private ConsumerController consumerController;
  95
  96     @Autowired
  97     private IcsSimulatorController icsSimulatorController;
  98
  99     @Autowired
 100     private KafkaTopicConsumers kafkaTopicConsumers;
 101
 102     private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create();
 103
 104     private static final Logger logger = LoggerFactory.getLogger(IntegrationWithKafka.class);
 105
 106     @LocalServerPort
 107     int localServerHttpPort;
 108
 109     static class TestApplicationConfig extends ApplicationConfig {
 110         @Override
 111         public String getIcsBaseUrl() {
 112             return thisProcessUrl();
 113         }
 114
 115         @Override
 116         public String getDmaapBaseUrl() {
 117             return thisProcessUrl();
 118         }
 119
 120         @Override
 121         public String getSelfUrl() {
 122             return thisProcessUrl();
 123         }
 124
 125         private String thisProcessUrl() {
 126             final String url = "https://localhost:" + getLocalServerHttpPort();
 127             return url;
 128         }
 129     }
 130
 131     /**
 132      * Overrides the BeanFactory.
 133      */
 134     @TestConfiguration
 135     static class TestBeanFactory extends BeanFactory {
 136
 137         @Override
 138         @Bean
 139         public ServletWebServerFactory servletContainer() {
 140             return new TomcatServletWebServerFactory();
 141         }
 142
 143         @Override
 144         @Bean
 145         public ApplicationConfig getApplicationConfig() {
 146             TestApplicationConfig cfg = new TestApplicationConfig();
 147             return cfg;
 148         }
 149     }
 150
 151     @AfterEach
 152     void reset() {
 153         this.consumerController.testResults.reset();
 154         this.icsSimulatorController.testResults.reset();
 155         this.jobs.clear();
 156     }
 157
 158     private AsyncRestClient restClient(boolean useTrustValidation) {
 159         WebClientConfig config = this.applicationConfig.getWebClientConfig();
 160         HttpProxyConfig httpProxyConfig = ImmutableHttpProxyConfig.builder() //
 161                 .httpProxyHost("") //
 162                 .httpProxyPort(0) //
 163                 .build();
 164         config = ImmutableWebClientConfig.builder() //
 165                 .keyStoreType(config.keyStoreType()) //
 166                 .keyStorePassword(config.keyStorePassword()) //
 167                 .keyStore(config.keyStore()) //
 168                 .keyPassword(config.keyPassword()) //
 169                 .isTrustStoreUsed(useTrustValidation) //
 170                 .trustStore(config.trustStore()) //
 171                 .trustStorePassword(config.trustStorePassword()) //
 172                 .httpProxyConfig(httpProxyConfig).build();
 173
 174         AsyncRestClientFactory restClientFactory = new AsyncRestClientFactory(config);
 175         return restClientFactory.createRestClientNoHttpProxy(baseUrl());
 176     }
 177
 178     private AsyncRestClient restClient() {
 179         return restClient(false);
 180     }
 181
 182     private String baseUrl() {
 183         return "https://localhost:" + this.applicationConfig.getLocalServerHttpPort();
 184     }
 185
 186     private static Object jobParametersAsJsonObject(String filter, long maxTimeMiliseconds, int maxSize,
 187             int maxConcurrency) {
 188         Job.Parameters param =
 189                 new Job.Parameters(filter, new Job.BufferTimeout(maxSize, maxTimeMiliseconds), maxConcurrency);
 190         String str = gson.toJson(param);
 191         return jsonObject(str);
 192     }
 193
 194     private static Object jsonObject(String json) {
 195         try {
 196             return JsonParser.parseString(json).getAsJsonObject();
 197         } catch (Exception e) {
 198             throw new NullPointerException(e.toString());
 199         }
 200     }
 201
 202     ConsumerJobInfo consumerJobInfo(String filter, Duration maxTime, int maxSize, int maxConcurrency) {
 203         try {
 204             String targetUri = baseUrl() + ConsumerController.CONSUMER_TARGET_URL;
 205             return new ConsumerJobInfo(TYPE_ID,
 206                     jobParametersAsJsonObject(filter, maxTime.toMillis(), maxSize, maxConcurrency), "owner", targetUri,
 207                     "");
 208         } catch (Exception e) {
 209             return null;
 210         }
 211     }
 212
 213     private SenderOptions<Integer, String> senderOptions() {
 214         String bootstrapServers = this.applicationConfig.getKafkaBootStrapServers();
 215
 216         Map<String, Object> props = new HashMap<>();
 217         props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
 218         props.put(ProducerConfig.CLIENT_ID_CONFIG, "sample-producerx");
 219         props.put(ProducerConfig.ACKS_CONFIG, "all");
 220         props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
 221         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
 222         return SenderOptions.create(props);
 223     }
 224
 225     private SenderRecord<Integer, String, Integer> senderRecord(String data) {
 226         final InfoType infoType = this.types.get(TYPE_ID);
 227         int key = 1;
 228         int correlationMetadata = 2;
 229         return SenderRecord.create(new ProducerRecord<>(infoType.getKafkaInputTopic(), key, data), correlationMetadata);
 230     }
 231
 232     private void sendDataToStream(Flux<SenderRecord<Integer, String, Integer>> dataToSend) {
 233         final KafkaSender<Integer, String> sender = KafkaSender.create(senderOptions());
 234
 235         sender.send(dataToSend) //
 236                 .doOnError(e -> logger.error("Send failed", e)) //
 237                 .blockLast();
 238
 239         sender.close();
 240
 241     }
 242
 243     private void verifiedReceivedByConsumer(String... strings) {
 244         ConsumerController.TestResults consumer = this.consumerController.testResults;
 245         await().untilAsserted(() -> assertThat(consumer.receivedBodies.size()).isEqualTo(strings.length));
 246         for (String s : strings) {
 247             assertTrue(consumer.hasReceived(s));
 248         }
 249     }
 250
 251     @Test
 252     void simpleCase() throws InterruptedException {
 253         final String JOB_ID = "ID";
 254
 255         // Register producer, Register types
 256         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 257         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 258
 259         this.icsSimulatorController.addJob(consumerJobInfo(null, Duration.ZERO, 0, 1), JOB_ID, restClient());
 260         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(1));
 261
 262         Thread.sleep(4000);
 263         var dataToSend = Flux.just(senderRecord("Message"));
 264         sendDataToStream(dataToSend);
 265
 266         verifiedReceivedByConsumer("Message");
 267
 268         this.icsSimulatorController.deleteJob(JOB_ID, restClient());
 269
 270         await().untilAsserted(() -> assertThat(this.jobs.size()).isZero());
 271         await().untilAsserted(() -> assertThat(this.kafkaTopicConsumers.getConsumers().keySet()).isEmpty());
 272     }
 273
 274     @Test
 275     void kafkaIntegrationTest() throws Exception {
 276         final String JOB_ID1 = "ID1";
 277         final String JOB_ID2 = "ID2";
 278
 279         // Register producer, Register types
 280         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 281         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 282
 283         // Create two jobs. One buffering and one with a filter
 284         this.icsSimulatorController.addJob(consumerJobInfo(null, Duration.ofMillis(400), 10, 20), JOB_ID1,
 285                 restClient());
 286         this.icsSimulatorController.addJob(consumerJobInfo("^Message_1$", Duration.ZERO, 0, 1), JOB_ID2, restClient());
 287
 288         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(2));
 289
 290         Thread.sleep(2000);
 291         var dataToSend = Flux.range(1, 3).map(i -> senderRecord("Message_" + i)); // Message_1, Message_2 etc.
 292         sendDataToStream(dataToSend);
 293
 294         verifiedReceivedByConsumer("Message_1", "[\"Message_1\", \"Message_2\", \"Message_3\"]");
 295
 296         // Delete the jobs
 297         this.icsSimulatorController.deleteJob(JOB_ID1, restClient());
 298         this.icsSimulatorController.deleteJob(JOB_ID2, restClient());
 299
 300         await().untilAsserted(() -> assertThat(this.jobs.size()).isZero());
 301         await().untilAsserted(() -> assertThat(this.kafkaTopicConsumers.getConsumers().keySet()).isEmpty());
 302     }
 303
 304     @Test
 305     void kafkaIOverflow() throws InterruptedException {
 306         final String JOB_ID1 = "ID1";
 307         final String JOB_ID2 = "ID2";
 308
 309         // Register producer, Register types
 310         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
 311         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
 312
 313         // Create two jobs.
 314         this.icsSimulatorController.addJob(consumerJobInfo(null, Duration.ofMillis(400), 1000, 1), JOB_ID1,
 315                 restClient());
 316         this.icsSimulatorController.addJob(consumerJobInfo(null, Duration.ZERO, 0, 1), JOB_ID2, restClient());
 317
 318         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(2));
 319
 320         var dataToSend = Flux.range(1, 1000000).map(i -> senderRecord("Message_" + i)); // Message_1, Message_2 etc.
 321         sendDataToStream(dataToSend); // this should overflow
 322
 323         KafkaJobDataConsumer consumer = kafkaTopicConsumers.getConsumers().get(TYPE_ID).iterator().next();
 324         await().untilAsserted(() -> assertThat(consumer.isRunning()).isFalse());
 325         this.consumerController.testResults.reset();
 326
 327         this.icsSimulatorController.deleteJob(JOB_ID2, restClient()); // Delete one job
 328         kafkaTopicConsumers.restartNonRunningTopics();
 329         Thread.sleep(1000); // Restarting the input seems to take some asynch time
 330
 331         dataToSend = Flux.just(senderRecord("Howdy\""));
 332         sendDataToStream(dataToSend);
 333
 334         verifiedReceivedByConsumer("[\"Howdy\\\"\"]");
 335
 336         // Delete the jobs
 337         this.icsSimulatorController.deleteJob(JOB_ID1, restClient());
 338         this.icsSimulatorController.deleteJob(JOB_ID2, restClient());
 339
 340         await().untilAsserted(() -> assertThat(this.jobs.size()).isZero());
 341         await().untilAsserted(() -> assertThat(this.kafkaTopicConsumers.getConsumers().keySet()).isEmpty());
 342     }
 343
 344 }