Changed so that registration never gives up
[nonrtric/plt/ranpm.git] / pmproducer / src / test / java / org / oran / pmproducer / ApplicationTest.java
1 /*-
2  * ========================LICENSE_START=================================
3  * O-RAN-SC
4  * %%
5  * Copyright (C) 2023 Nordix Foundation
6  * %%
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ========================LICENSE_END===================================
19  */
20
21 package org.oran.pmproducer;
22
23 import static org.assertj.core.api.Assertions.assertThat;
24 import static org.awaitility.Awaitility.await;
25 import static org.junit.jupiter.api.Assertions.assertTrue;
26 import static org.mockito.ArgumentMatchers.any;
27 import static org.mockito.Mockito.doReturn;
28 import static org.mockito.Mockito.spy;
29 import static org.mockito.Mockito.times;
30 import static org.mockito.Mockito.verify;
31 import static org.mockito.Mockito.when;
32
33 import com.google.gson.JsonParser;
34
35 import java.io.FileOutputStream;
36 import java.io.IOException;
37 import java.io.PrintStream;
38 import java.lang.invoke.MethodHandles;
39 import java.nio.file.Files;
40 import java.nio.file.Path;
41 import java.nio.file.Paths;
42 import java.time.Duration;
43 import java.time.OffsetDateTime;
44 import java.util.List;
45
46 import org.apache.kafka.clients.consumer.ConsumerRecord;
47 import org.json.JSONObject;
48 import org.junit.jupiter.api.AfterEach;
49 import org.junit.jupiter.api.BeforeEach;
50 import org.junit.jupiter.api.MethodOrderer;
51 import org.junit.jupiter.api.Test;
52 import org.junit.jupiter.api.TestMethodOrder;
53 import org.mockito.ArgumentCaptor;
54 import org.oran.pmproducer.clients.AsyncRestClient;
55 import org.oran.pmproducer.clients.AsyncRestClientFactory;
56 import org.oran.pmproducer.clients.SecurityContext;
57 import org.oran.pmproducer.configuration.ApplicationConfig;
58 import org.oran.pmproducer.configuration.WebClientConfig;
59 import org.oran.pmproducer.configuration.WebClientConfig.HttpProxyConfig;
60 import org.oran.pmproducer.controllers.ProducerCallbacksController;
61 import org.oran.pmproducer.datastore.DataStore;
62 import org.oran.pmproducer.datastore.DataStore.Bucket;
63 import org.oran.pmproducer.filter.FilteredData;
64 import org.oran.pmproducer.filter.PmReport;
65 import org.oran.pmproducer.filter.PmReportFilter;
66 import org.oran.pmproducer.filter.PmReportFilter.FilterData;
67 import org.oran.pmproducer.r1.ConsumerJobInfo;
68 import org.oran.pmproducer.r1.ProducerJobInfo;
69 import org.oran.pmproducer.repository.InfoType;
70 import org.oran.pmproducer.repository.InfoTypes;
71 import org.oran.pmproducer.repository.Job;
72 import org.oran.pmproducer.repository.Job.Parameters;
73 import org.oran.pmproducer.repository.Job.Parameters.KafkaDeliveryInfo;
74 import org.oran.pmproducer.repository.Jobs;
75 import org.oran.pmproducer.repository.Jobs.JobGroup;
76 import org.oran.pmproducer.tasks.JobDataDistributor;
77 import org.oran.pmproducer.tasks.NewFileEvent;
78 import org.oran.pmproducer.tasks.ProducerRegstrationTask;
79 import org.oran.pmproducer.tasks.TopicListener;
80 import org.oran.pmproducer.tasks.TopicListeners;
81 import org.slf4j.Logger;
82 import org.slf4j.LoggerFactory;
83 import org.springframework.beans.factory.annotation.Autowired;
84 import org.springframework.boot.test.context.SpringBootTest;
85 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
86 import org.springframework.boot.test.context.TestConfiguration;
87 import org.springframework.boot.test.web.server.LocalServerPort;
88 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
89 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
90 import org.springframework.context.annotation.Bean;
91 import org.springframework.http.HttpStatus;
92 import org.springframework.http.MediaType;
93 import org.springframework.http.ResponseEntity;
94 import org.springframework.test.context.TestPropertySource;
95 import org.springframework.web.reactive.function.client.WebClientRequestException;
96 import org.springframework.web.reactive.function.client.WebClientResponseException;
97
98 import reactor.core.publisher.Flux;
99 import reactor.core.publisher.Mono;
100 import reactor.test.StepVerifier;
101
102 @TestMethodOrder(MethodOrderer.MethodName.class)
103 @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
104 @TestPropertySource(properties = { //
105         "server.ssl.key-store=./config/keystore.jks", //
106         "app.webclient.trust-store=./config/truststore.jks", //
107         "app.webclient.trust-store-used=true", //
108         "app.configuration-filepath=./src/test/resources/test_application_configuration.json", //
109         "app.pm-files-path=/tmp/dmaapadaptor", //
110         "app.s3.endpointOverride=" //
111 })
112 class ApplicationTest {
113
114     @Autowired
115     private ApplicationConfig applicationConfig;
116
117     @Autowired
118     private Jobs jobs;
119
120     @Autowired
121     private InfoTypes types;
122
123     @Autowired
124     private IcsSimulatorController icsSimulatorController;
125
126     @Autowired
127     TopicListeners topicListeners;
128
129     @Autowired
130     ProducerRegstrationTask producerRegistrationTask;
131
132     @Autowired
133     private SecurityContext securityContext;
134
135     private com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create();
136
137     @LocalServerPort
138     int localServerHttpPort;
139
140     private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
141
142     static class TestApplicationConfig extends ApplicationConfig {
143
144         @Override
145         public String getIcsBaseUrl() {
146             return thisProcessUrl();
147         }
148
149         @Override
150         public String getSelfUrl() {
151             return thisProcessUrl();
152         }
153
154         private String thisProcessUrl() {
155             final String url = "https://localhost:" + getLocalServerHttpPort();
156             return url;
157         }
158     }
159
160     /**
161      * Overrides the BeanFactory.
162      */
163     @TestConfiguration
164     static class TestBeanFactory extends BeanFactory {
165
166         @Override
167         @Bean
168         public ServletWebServerFactory servletContainer() {
169             return new TomcatServletWebServerFactory();
170         }
171
172         // @Override
173         @Bean
174         public ApplicationConfig getApplicationConfig() {
175             TestApplicationConfig cfg = new TestApplicationConfig();
176             return cfg;
177         }
178     }
179
180     @BeforeEach
181     public void init() {
182         this.applicationConfig.setLocalServerHttpPort(this.localServerHttpPort);
183         assertThat(this.jobs.size()).isZero();
184
185         DataStore fileStore = this.dataStore();
186         fileStore.create(DataStore.Bucket.FILES).block();
187         fileStore.create(DataStore.Bucket.LOCKS).block();
188     }
189
190     private DataStore dataStore() {
191         return DataStore.create(this.applicationConfig);
192     }
193
194     @AfterEach
195     void reset() {
196
197         for (Job job : this.jobs.getAll()) {
198             this.icsSimulatorController.deleteJob(job.getId(), restClient());
199         }
200         await().untilAsserted(() -> assertThat(this.jobs.size()).isZero());
201
202         this.icsSimulatorController.testResults.reset();
203
204         DataStore fileStore = DataStore.create(applicationConfig);
205         fileStore.deleteBucket(Bucket.FILES);
206         fileStore.deleteBucket(Bucket.LOCKS);
207
208     }
209
210     private AsyncRestClient restClient(boolean useTrustValidation) {
211         WebClientConfig config = this.applicationConfig.getWebClientConfig();
212         HttpProxyConfig httpProxyConfig = HttpProxyConfig.builder() //
213                 .httpProxyHost("") //
214                 .httpProxyPort(0) //
215                 .build();
216         config = WebClientConfig.builder() //
217                 .keyStoreType(config.getKeyStoreType()) //
218                 .keyStorePassword(config.getKeyStorePassword()) //
219                 .keyStore(config.getKeyStore()) //
220                 .keyPassword(config.getKeyPassword()) //
221                 .isTrustStoreUsed(useTrustValidation) //
222                 .trustStore(config.getTrustStore()) //
223                 .trustStorePassword(config.getTrustStorePassword()) //
224                 .httpProxyConfig(httpProxyConfig).build();
225
226         AsyncRestClientFactory restClientFactory = new AsyncRestClientFactory(config, securityContext);
227         return restClientFactory.createRestClientNoHttpProxy(baseUrl());
228     }
229
230     private AsyncRestClient restClient() {
231         return restClient(false);
232     }
233
234     private String baseUrl() {
235         return "https://localhost:" + this.applicationConfig.getLocalServerHttpPort();
236     }
237
238     private String quote(String str) {
239         final String q = "\"";
240         return q + str.replace(q, "\\\"") + q;
241     }
242
243     private Object toJson(String json) {
244         try {
245             return JsonParser.parseString(json).getAsJsonObject();
246         } catch (Exception e) {
247             throw new NullPointerException(e.toString());
248         }
249     }
250
251     private ConsumerJobInfo consumerJobInfo(String typeId, String infoJobId, Object filter) {
252         try {
253             return new ConsumerJobInfo(typeId, filter, "owner", "");
254         } catch (Exception e) {
255             return null;
256         }
257     }
258
259     private void waitForRegistration() {
260         producerRegistrationTask.registerTypesAndProducer().block();
261         // Register producer, Register types
262         await().untilAsserted(() -> assertThat(icsSimulatorController.testResults.registrationInfo).isNotNull());
263         producerRegistrationTask.registerTypesAndProducer().block();
264
265         assertThat(icsSimulatorController.testResults.registrationInfo.supportedTypeIds).hasSize(this.types.size());
266         assertThat(producerRegistrationTask.isRegisteredInIcs()).isTrue();
267         assertThat(icsSimulatorController.testResults.types).hasSize(this.types.size());
268     }
269
270     @Test
271     void generateApiDoc() throws IOException {
272         String url = "https://localhost:" + applicationConfig.getLocalServerHttpPort() + "/v3/api-docs";
273         ResponseEntity<String> resp = restClient().getForEntity(url).block();
274         assertThat(resp.getStatusCode()).isEqualTo(HttpStatus.OK);
275         JSONObject jsonObj = new JSONObject(resp.getBody());
276         assertThat(jsonObj.remove("servers")).isNotNull();
277
278         String indented = (jsonObj).toString(4);
279         String docDir = "api/";
280         Files.createDirectories(Paths.get(docDir));
281         try (PrintStream out = new PrintStream(new FileOutputStream(docDir + "api.json"))) {
282             out.print(indented);
283         }
284     }
285
286     @Test
287     void testTrustValidation() throws IOException {
288         String url = "https://localhost:" + applicationConfig.getLocalServerHttpPort() + "/v3/api-docs";
289         ResponseEntity<String> resp = restClient(true).getForEntity(url).block();
290         assertThat(resp.getStatusCode()).isEqualTo(HttpStatus.OK);
291     }
292
293     @Test
294     void testResponseCodes() throws Exception {
295         String supervisionUrl = baseUrl() + ProducerCallbacksController.SUPERVISION_URL;
296         ResponseEntity<String> resp = restClient().getForEntity(supervisionUrl).block();
297         assertThat(resp.getStatusCode()).isEqualTo(HttpStatus.OK);
298
299         String jobUrl = baseUrl() + ProducerCallbacksController.JOB_URL;
300         resp = restClient().deleteForEntity(jobUrl + "/junk").block();
301         assertThat(resp.getStatusCode()).isEqualTo(HttpStatus.OK);
302
303         ProducerJobInfo info = new ProducerJobInfo(null, "id", "typeId", "owner", "lastUpdated");
304         String body = gson.toJson(info);
305         testErrorCode(restClient().post(jobUrl, body, MediaType.APPLICATION_JSON), HttpStatus.NOT_FOUND,
306                 "Could not find type");
307     }
308
309     @Test
310     void testFiltering() {
311         String path = "./src/test/resources/pm_report.json.gz";
312         DataStore fs = DataStore.create(this.applicationConfig);
313         fs.copyFileTo(Path.of(path), "pm_report.json.gz");
314
315         InfoType infoType = this.types.getAll().iterator().next();
316         TopicListener listener = spy(new TopicListener(this.applicationConfig, infoType));
317         NewFileEvent event = NewFileEvent.builder().filename("pm_report.json.gz").build();
318         ConsumerRecord<byte[], byte[]> cr = new ConsumerRecord<>("", 0, 0, new byte[0], gson.toJson(event).getBytes());
319         when(listener.receiveFromKafka(any())).thenReturn(Flux.just(cr));
320
321         KafkaDeliveryInfo deliveryInfo = KafkaDeliveryInfo.builder().topic("topic").bootStrapServers("").build();
322         JobGroup jobGroup = new JobGroup(infoType, deliveryInfo);
323         jobGroup.add(new Job("id", infoType, "owner", "lastUpdated",
324                 Parameters.builder().filter(new FilterData()).build(), this.applicationConfig));
325         JobDataDistributor distributor = spy(new JobDataDistributor(jobGroup, this.applicationConfig));
326
327         doReturn(Mono.just("")).when(distributor).sendToClient(any());
328
329         distributor.start(listener.getFlux());
330
331         {
332             ArgumentCaptor<FilteredData> captor = ArgumentCaptor.forClass(FilteredData.class);
333             verify(distributor).sendToClient(captor.capture());
334             FilteredData data = captor.getValue();
335             PmReport report = PmReportFilter.parse(new String(data.value));
336             assertThat(report.event.getCommonEventHeader().getSourceName()).isEqualTo("O-DU-1122");
337         }
338     }
339
340     @Test
341     void testFilteringHistoricalData() {
342         DataStore fileStore = DataStore.create(this.applicationConfig);
343         fileStore.copyFileTo(Path.of("./src/test/resources/pm_report.json"),
344                 "O-DU-1122/A20000626.2315+0200-2330+0200_HTTPS-6-73.json").block();
345
346         InfoType infoType = this.types.getAll().iterator().next();
347         TopicListener listener = spy(new TopicListener(this.applicationConfig, infoType));
348         Flux<ConsumerRecord<byte[], byte[]>> flux = Flux.just("") //
349                 .delayElements(Duration.ofSeconds(10)).flatMap(s -> Flux.empty());
350         when(listener.receiveFromKafka(any())).thenReturn(flux);
351
352         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
353         filterData.getSourceNames().add("O-DU-1122");
354         filterData.setPmRopStartTime("1999-12-27T10:50:44.000-08:00");
355         filterData.setPmRopEndTime(OffsetDateTime.now().toString());
356         KafkaDeliveryInfo deliveryInfo = KafkaDeliveryInfo.builder().topic("topic").bootStrapServers("").build();
357         JobGroup jobGroup = new JobGroup(infoType, deliveryInfo);
358         Parameters params = Parameters.builder().filter(filterData).build();
359         Job job = new Job("id", infoType, "owner", "lastUpdated", params, this.applicationConfig);
360         jobGroup.add(job);
361         JobDataDistributor distributor = spy(new JobDataDistributor(jobGroup, this.applicationConfig));
362
363         doReturn(Mono.just("")).when(distributor).sendToClient(any());
364
365         distributor.start(listener.getFlux());
366
367         {
368             ArgumentCaptor<FilteredData> captor = ArgumentCaptor.forClass(FilteredData.class);
369             verify(distributor, times(2)).sendToClient(captor.capture());
370             List<FilteredData> data = captor.getAllValues();
371             assertThat(data).hasSize(2);
372             PmReport report = PmReportFilter.parse(new String(data.get(0).value));
373             assertThat(report.event.getCommonEventHeader().getSourceName()).isEqualTo("O-DU-1122");
374             assertThat(new String(data.get(1).value)).isEqualTo("{}");
375         }
376     }
377
378     @Test
379     void testCreateJob() throws Exception {
380         // Create a job
381         final String JOB_ID = "ID";
382
383         // Register producer, Register types
384         waitForRegistration();
385
386         assertThat(this.topicListeners.getTopicListeners()).hasSize(1);
387
388         // Create a job with a PM filter
389         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
390
391         filterData.addMeasTypes("UtranCell", "succImmediateAssignProcs");
392         filterData.getMeasObjInstIds().add("UtranCell=Gbg-997");
393         filterData.getSourceNames().add("O-DU-1122");
394         filterData.getMeasuredEntityDns().add("ManagedElement=RNC-Gbg-1");
395         Job.Parameters param = Job.Parameters.builder() //
396                 .filter(filterData).deliveryInfo(KafkaDeliveryInfo.builder().bootStrapServers("").topic("").build()) //
397                 .build();
398
399         String paramJson = gson.toJson(param);
400         System.out.println(paramJson);
401         ConsumerJobInfo jobInfo = consumerJobInfo("PmDataOverKafka", "EI_PM_JOB_ID", toJson(paramJson));
402
403         this.icsSimulatorController.addJob(jobInfo, JOB_ID, restClient());
404         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(1));
405
406         assertThat(this.topicListeners.getDataDistributors().keySet()).hasSize(1);
407     }
408
409     @Test
410     void testPmFilteringKafka() throws Exception {
411         // Test that the schema for kafka and pm filtering is OK.
412
413         // Create a job
414         final String JOB_ID = "ID";
415
416         // Register producer, Register types
417         waitForRegistration();
418
419         // Create a job with a PM filter
420         PmReportFilter.FilterData filterData = new PmReportFilter.FilterData();
421         filterData.addMeasTypes("ManagedElement", "succImmediateAssignProcs");
422         Job.Parameters param = Job.Parameters.builder() //
423                 .filter(filterData).deliveryInfo(KafkaDeliveryInfo.builder().bootStrapServers("").topic("").build()) //
424                 .build();
425
426         String paramJson = gson.toJson(param);
427
428         ConsumerJobInfo jobInfo = consumerJobInfo("PmDataOverKafka", "EI_PM_JOB_ID", toJson(paramJson));
429         this.icsSimulatorController.addJob(jobInfo, JOB_ID, restClient());
430         await().untilAsserted(() -> assertThat(this.jobs.size()).isEqualTo(1));
431     }
432
433     @Test
434     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
435     void testZZActuator() throws Exception {
436         // The test must be run last, hence the "ZZ" in the name. All succeeding tests
437         // will fail.
438         AsyncRestClient client = restClient();
439         client.post("/actuator/loggers/org.oran.pmproducer", "{\"configuredLevel\":\"trace\"}").block();
440         String resp = client.get("/actuator/loggers/org.oran.pmproducer").block();
441         assertThat(resp).contains("TRACE");
442         client.post("/actuator/loggers/org.springframework.boot.actuate", "{\"configuredLevel\":\"trace\"}").block();
443         // This will stop the web server and all coming tests will fail.
444         client.post("/actuator/shutdown", "").block();
445         Thread.sleep(1000);
446
447         String url = "https://localhost:" + applicationConfig.getLocalServerHttpPort() + "/v3/api-docs";
448         StepVerifier.create(restClient().get(url)) // Any call
449                 .expectSubscription() //
450                 .expectErrorMatches(t -> t instanceof WebClientRequestException) //
451                 .verify();
452     }
453
454     public static void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains) {
455         testErrorCode(request, expStatus, responseContains, true);
456     }
457
458     public static void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains,
459             boolean expectApplicationProblemJsonMediaType) {
460         StepVerifier.create(request) //
461                 .expectSubscription() //
462                 .expectErrorMatches(
463                         t -> checkWebClientError(t, expStatus, responseContains, expectApplicationProblemJsonMediaType)) //
464                 .verify();
465     }
466
467     private static boolean checkWebClientError(Throwable throwable, HttpStatus expStatus, String responseContains,
468             boolean expectApplicationProblemJsonMediaType) {
469         assertTrue(throwable instanceof WebClientResponseException);
470         WebClientResponseException responseException = (WebClientResponseException) throwable;
471         assertThat(responseException.getStatusCode()).isEqualTo(expStatus);
472         assertThat(responseException.getResponseBodyAsString()).contains(responseContains);
473         if (expectApplicationProblemJsonMediaType) {
474             assertThat(responseException.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_PROBLEM_JSON);
475         }
476         return true;
477     }
478 }