2 * ========================LICENSE_START=================================
5 * Copyright (C) 2021 Nordix Foundation
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ========================LICENSE_END===================================
21 package org.oran.dmaapadapter;
23 import static org.assertj.core.api.Assertions.assertThat;
24 import static org.awaitility.Awaitility.await;
25 import static org.junit.jupiter.api.Assertions.assertTrue;
27 import com.google.gson.JsonParser;
29 import java.time.Duration;
30 import java.util.HashMap;
33 import org.apache.kafka.clients.producer.ProducerConfig;
34 import org.apache.kafka.clients.producer.ProducerRecord;
35 import org.apache.kafka.common.serialization.IntegerSerializer;
36 import org.apache.kafka.common.serialization.StringSerializer;
37 import org.junit.jupiter.api.AfterEach;
38 import org.junit.jupiter.api.Test;
39 import org.junit.jupiter.api.extension.ExtendWith;
40 import org.oran.dmaapadapter.clients.AsyncRestClient;
41 import org.oran.dmaapadapter.clients.AsyncRestClientFactory;
42 import org.oran.dmaapadapter.configuration.ApplicationConfig;
43 import org.oran.dmaapadapter.configuration.ImmutableHttpProxyConfig;
44 import org.oran.dmaapadapter.configuration.ImmutableWebClientConfig;
45 import org.oran.dmaapadapter.configuration.WebClientConfig;
46 import org.oran.dmaapadapter.configuration.WebClientConfig.HttpProxyConfig;
47 import org.oran.dmaapadapter.r1.ConsumerJobInfo;
48 import org.oran.dmaapadapter.repository.InfoType;
49 import org.oran.dmaapadapter.repository.InfoTypes;
50 import org.oran.dmaapadapter.repository.Job;
51 import org.oran.dmaapadapter.repository.Jobs;
52 import org.oran.dmaapadapter.tasks.KafkaJobDataConsumer;
53 import org.oran.dmaapadapter.tasks.KafkaTopicConsumers;
54 import org.slf4j.Logger;
55 import org.slf4j.LoggerFactory;
56 import org.springframework.beans.factory.annotation.Autowired;
57 import org.springframework.boot.test.context.SpringBootTest;
58 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
59 import org.springframework.boot.test.context.TestConfiguration;
60 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
61 import org.springframework.boot.web.server.LocalServerPort;
62 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
63 import org.springframework.context.annotation.Bean;
64 import org.springframework.test.context.TestPropertySource;
65 import org.springframework.test.context.junit.jupiter.SpringExtension;
67 import reactor.core.publisher.Flux;
68 import reactor.kafka.sender.KafkaSender;
69 import reactor.kafka.sender.SenderOptions;
70 import reactor.kafka.sender.SenderRecord;
72 @SuppressWarnings("java:S3577") // Rename class
73 @ExtendWith(SpringExtension.class)
74 @SpringBootTest(webEnvironment = WebEnvironment.DEFINED_PORT)
75 @TestPropertySource(properties = { //
76 "server.ssl.key-store=./config/keystore.jks", //
77 "app.webclient.trust-store=./config/truststore.jks", //
78 "app.configuration-filepath=./src/test/resources/test_application_configuration_kafka.json"//
80 class IntegrationWithKafka {
83 private ApplicationConfig applicationConfig;
89 private InfoTypes types;
92 private ConsumerController consumerController;
95 private EcsSimulatorController ecsSimulatorController;
98 private KafkaTopicConsumers kafkaTopicConsumers;
100 private com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create();
102 private static final Logger logger = LoggerFactory.getLogger(IntegrationWithKafka.class);
105 int localServerHttpPort;
107 static class TestApplicationConfig extends ApplicationConfig {
109 public String getEcsBaseUrl() {
110 return thisProcessUrl();
114 public String getDmaapBaseUrl() {
115 return thisProcessUrl();
119 public String getSelfUrl() {
120 return thisProcessUrl();
123 private String thisProcessUrl() {
124 final String url = "https://localhost:" + getLocalServerHttpPort();
130 * Overrides the BeanFactory.
133 static class TestBeanFactory extends BeanFactory {
137 public ServletWebServerFactory servletContainer() {
138 return new TomcatServletWebServerFactory();
143 public ApplicationConfig getApplicationConfig() {
144 TestApplicationConfig cfg = new TestApplicationConfig();
151 this.consumerController.testResults.reset();
152 this.ecsSimulatorController.testResults.reset();
156 private AsyncRestClient restClient(boolean useTrustValidation) {
157 WebClientConfig config = this.applicationConfig.getWebClientConfig();
158 HttpProxyConfig httpProxyConfig = ImmutableHttpProxyConfig.builder() //
159 .httpProxyHost("") //
162 config = ImmutableWebClientConfig.builder() //
163 .keyStoreType(config.keyStoreType()) //
164 .keyStorePassword(config.keyStorePassword()) //
165 .keyStore(config.keyStore()) //
166 .keyPassword(config.keyPassword()) //
167 .isTrustStoreUsed(useTrustValidation) //
168 .trustStore(config.trustStore()) //
169 .trustStorePassword(config.trustStorePassword()) //
170 .httpProxyConfig(httpProxyConfig).build();
172 AsyncRestClientFactory restClientFactory = new AsyncRestClientFactory(config);
173 return restClientFactory.createRestClientNoHttpProxy(baseUrl());
176 private AsyncRestClient restClient() {
177 return restClient(false);
180 private String baseUrl() {
181 return "https://localhost:" + this.applicationConfig.getLocalServerHttpPort();
184 private Object jobParametersAsJsonObject(String filter, long maxTimeMiliseconds, int maxSize, int maxConcurrency) {
185 Job.Parameters param =
186 new Job.Parameters(filter, new Job.BufferTimeout(maxSize, maxTimeMiliseconds), maxConcurrency);
187 String str = gson.toJson(param);
188 return jsonObject(str);
191 private Object jsonObject(String json) {
193 return JsonParser.parseString(json).getAsJsonObject();
194 } catch (Exception e) {
195 throw new NullPointerException(e.toString());
199 private ConsumerJobInfo consumerJobInfo(String filter, Duration maxTime, int maxSize, int maxConcurrency) {
201 InfoType type = this.types.getAll().iterator().next();
202 String typeId = type.getId();
203 String targetUri = baseUrl() + ConsumerController.CONSUMER_TARGET_URL;
204 return new ConsumerJobInfo(typeId,
205 jobParametersAsJsonObject(filter, maxTime.toMillis(), maxSize, maxConcurrency), "owner", targetUri,
207 } catch (Exception e) {
212 private SenderOptions<Integer, String> senderOptions() {
213 String bootstrapServers = this.applicationConfig.getKafkaBootStrapServers();
215 Map<String, Object> props = new HashMap<>();
216 props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
217 props.put(ProducerConfig.CLIENT_ID_CONFIG, "sample-producer");
218 props.put(ProducerConfig.ACKS_CONFIG, "all");
219 props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
220 props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
221 return SenderOptions.create(props);
224 private SenderRecord<Integer, String, Integer> senderRecord(String data, int i) {
225 final InfoType infoType = this.types.getAll().iterator().next();
226 return SenderRecord.create(new ProducerRecord<>(infoType.getKafkaInputTopic(), i, data + i), i);
229 private void sendDataToStream(Flux<SenderRecord<Integer, String, Integer>> dataToSend) {
230 final KafkaSender<Integer, String> sender = KafkaSender.create(senderOptions());
232 sender.send(dataToSend) //
233 .doOnError(e -> logger.error("Send failed", e)) //
238 private void verifiedReceivedByConsumer(String... strings) {
239 ConsumerController.TestResults consumer = this.consumerController.testResults;
240 await().untilAsserted(() -> assertThat(consumer.receivedBodies.size()).isEqualTo(strings.length));
241 for (String s : strings) {
242 assertTrue(consumer.hasReceived(s));
247 void kafkaIntegrationTest() throws InterruptedException {
248 final String JOB_ID1 = "ID1";
249 final String JOB_ID2 = "ID2";
251 // Register producer, Register types
252 await().untilAsserted(() -> assertThat(ecsSimulatorController.testResults.registrationInfo).isNotNull());
253 assertThat(ecsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(1);
255 // Create two jobs. One buffering and one with a filter
256 this.ecsSimulatorController.addJob(consumerJobInfo(null, Duration.ofMillis(400), 1000, 20), JOB_ID1,
258 this.ecsSimulatorController.addJob(consumerJobInfo("^Message_1$", Duration.ZERO, 0, 1), JOB_ID2, restClient());
260 await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(2));
262 var dataToSend = Flux.range(1, 3).map(i -> senderRecord("Message_", i)); // Message_1, Message_2 etc.
263 sendDataToStream(dataToSend);
265 verifiedReceivedByConsumer("Message_1", "[\"Message_1\", \"Message_2\", \"Message_3\"]");
267 // Just for testing quoting
268 this.consumerController.testResults.reset();
269 dataToSend = Flux.just(senderRecord("Message\"_", 1));
270 sendDataToStream(dataToSend);
271 verifiedReceivedByConsumer("[\"Message\\\"_1\"]");
274 this.ecsSimulatorController.deleteJob(JOB_ID1, restClient());
275 this.ecsSimulatorController.deleteJob(JOB_ID2, restClient());
277 await().untilAsserted(() -> assertThat(this.jobs.size()).isZero());
278 await().untilAsserted(() -> assertThat(this.kafkaTopicConsumers.getConsumers()).isEmpty());
282 void kafkaIOverflow() throws InterruptedException {
283 final String JOB_ID1 = "ID1";
284 final String JOB_ID2 = "ID2";
286 // Register producer, Register types
287 await().untilAsserted(() -> assertThat(ecsSimulatorController.testResults.registrationInfo).isNotNull());
288 assertThat(ecsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(1);
291 this.ecsSimulatorController.addJob(consumerJobInfo(null, Duration.ZERO, 0, 1), JOB_ID1, restClient());
292 this.ecsSimulatorController.addJob(consumerJobInfo(null, Duration.ZERO, 0, 1), JOB_ID2, restClient());
294 await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(2));
296 var dataToSend = Flux.range(1, 1000000).map(i -> senderRecord("Message_", i)); // Message_1, Message_2 etc.
297 sendDataToStream(dataToSend); // this should overflow
299 KafkaJobDataConsumer consumer = kafkaTopicConsumers.getConsumers().values().iterator().next();
300 await().untilAsserted(() -> assertThat(consumer.isRunning()).isFalse());
301 this.consumerController.testResults.reset();
303 kafkaTopicConsumers.restartNonRunningTasks();
304 this.ecsSimulatorController.deleteJob(JOB_ID2, restClient()); // Delete one job
305 Thread.sleep(1000); // Restarting the input seems to take some asynch time
307 dataToSend = Flux.range(1, 1).map(i -> senderRecord("Howdy_", i));
308 sendDataToStream(dataToSend);
310 verifiedReceivedByConsumer("Howdy_1");