2 * ============LICENSE_START======================================================================
3 * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
4 * Copyright (C) 2020-2022 Nokia. All rights reserved.
5 * ===============================================================================================
6 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
7 * in compliance with the License. You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software distributed under the License
12 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
13 * or implied. See the License for the specific language governing permissions and limitations under
15 * ============LICENSE_END========================================================================
18 package org.onap.dcaegen2.collectors.datafile.tasks;
20 import static org.junit.jupiter.api.Assertions.assertEquals;
21 import static org.mockito.ArgumentMatchers.any;
22 import static org.mockito.Mockito.doReturn;
23 import static org.mockito.Mockito.doThrow;
24 import static org.mockito.Mockito.mock;
25 import static org.mockito.Mockito.spy;
26 import static org.mockito.Mockito.times;
27 import static org.mockito.Mockito.verify;
28 import static org.mockito.Mockito.verifyNoMoreInteractions;
29 import static org.mockito.Mockito.when;
31 import java.nio.file.Path;
32 import java.nio.file.Paths;
33 import java.time.Duration;
34 import java.util.ArrayList;
35 import java.util.List;
37 import org.junit.jupiter.api.BeforeAll;
38 import org.junit.jupiter.api.BeforeEach;
39 import org.junit.jupiter.api.Test;
40 import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
41 import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig;
42 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
43 import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
44 import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient;
45 import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
46 import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient;
47 import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient;
48 import org.onap.dcaegen2.collectors.datafile.model.Counters;
49 import org.onap.dcaegen2.collectors.datafile.model.FileData;
50 import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
51 import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage;
52 import reactor.test.StepVerifier;
54 public class FileCollectorTest {
56 final static String DATAFILE_TMPDIR = "/tmp/onap_datafile/";
57 private static final String PRODUCT_NAME = "NrRadio";
58 private static final String VENDOR_NAME = "Ericsson";
59 private static final int LAST_EPOCH_MICROSEC = 87457457;
60 private static final String SOURCE_NAME = "oteNB5309";
61 private static final int START_EPOCH_MICROSEC = 874575764;
62 private static final String TIME_ZONE_OFFSET = "UTC+05:00";
63 private static final String FTPES_SCHEME = "ftpes://";
64 private static final String SFTP_SCHEME = "sftp://";
65 private static final String HTTP_SCHEME = "http://";
66 private static final String HTTPS_SCHEME = "https://";
67 private static final String SERVER_ADDRESS = "192.168.0.101";
68 private static final int PORT_22 = 22;
69 private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz";
70 private static final Path LOCAL_FILE_LOCATION = Paths.get(DATAFILE_TMPDIR, SOURCE_NAME, PM_FILE_NAME);
71 private static final String REMOTE_FILE_LOCATION = "/ftp/rop/" + PM_FILE_NAME;
72 private static final String USER = "usr";
73 private static final String PWD = "pwd";
74 private static final String FTPES_LOCATION =
75 FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
77 private static final String FTPES_LOCATION_NO_PORT =
78 FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
79 private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
80 private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
82 private static final String HTTP_LOCATION =
83 HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
84 private static final String HTTP_LOCATION_NO_PORT =
85 HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
86 private static final String HTTPS_LOCATION =
87 HTTPS_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
88 private static final String HTTPS_LOCATION_NO_PORT =
89 HTTPS_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
91 private static final String GZIP_COMPRESSION = "gzip";
92 private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
93 private static final String FILE_FORMAT_VERSION = "V10";
94 private static final String CERTIFICATE_KEY_PASSWORD_PATH = "certificateKeyPassword";
95 private static final String TRUSTED_CA_PATH = "trustedCAPath";
96 private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword";
97 private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES";
98 private static final String FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
99 private static final String CHANGE_TYPE = "FileReady";
101 private static AppConfig appConfigMock = mock(AppConfig.class);
102 private static CertificateConfig certificateConfigMock = mock(CertificateConfig.class);
104 private FtpesClient ftpesClientMock = mock(FtpesClient.class);
106 private SftpClient sftpClientMock = mock(SftpClient.class);
108 private DfcHttpClient dfcHttpClientMock = mock(DfcHttpClient.class);
109 private DfcHttpsClient dfcHttpsClientMock = mock(DfcHttpsClient.class);
111 private Counters counters;
113 FileReadyMessage.Event event(String location) {
114 FileReadyMessage.MessageMetaData messageMetaData = FileReadyMessage.MessageMetaData.builder() //
115 .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
116 .sourceName(SOURCE_NAME) //
117 .startEpochMicrosec(START_EPOCH_MICROSEC) //
118 .timeZoneOffset(TIME_ZONE_OFFSET) //
119 .changeIdentifier(CHANGE_IDENTIFIER) //
120 .eventName("Noti_NrRadio-Ericsson_FileReady").build();
122 FileReadyMessage.FileInfo fileInfo = FileReadyMessage.FileInfo //
124 .fileFormatType(FILE_FORMAT_TYPE) //
125 .location(location) //
126 .fileFormatVersion(FILE_FORMAT_VERSION) //
127 .compression(GZIP_COMPRESSION) //
130 FileReadyMessage.ArrayOfNamedHashMap arrayOfNamedHashMap = FileReadyMessage.ArrayOfNamedHashMap //
131 .builder().name(PM_FILE_NAME) //
132 .hashMap(fileInfo).build();
134 List<FileReadyMessage.ArrayOfNamedHashMap> arrayOfNamedHashMapList = new ArrayList<>();
135 arrayOfNamedHashMapList.add(arrayOfNamedHashMap);
137 FileReadyMessage.NotificationFields notificationFields = FileReadyMessage.NotificationFields //
138 .builder().notificationFieldsVersion("notificationFieldsVersion") //
139 .changeType(CHANGE_TYPE).changeIdentifier(CHANGE_IDENTIFIER) //
140 .arrayOfNamedHashMap(arrayOfNamedHashMapList) //
143 return FileReadyMessage.Event.builder() //
144 .commonEventHeader(messageMetaData) //
145 .notificationFields(notificationFields).build();
148 private FileReadyMessage fileReadyMessage(String location) {
149 FileReadyMessage message = FileReadyMessage.builder() //
150 .event(event(location)) //
155 private FileData createFileData(String location) {
156 return FileData.createFileData(fileReadyMessage(location)).iterator().next();
159 private FilePublishInformation createExpectedFilePublishInformation(String location) {
160 return FilePublishInformation.builder() //
161 .productName(PRODUCT_NAME) //
162 .vendorName(VENDOR_NAME) //
163 .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
164 .sourceName(SOURCE_NAME) //
165 .startEpochMicrosec(START_EPOCH_MICROSEC) //
166 .timeZoneOffset(TIME_ZONE_OFFSET) //
167 .name(SOURCE_NAME + "/" + PM_FILE_NAME) //
168 .compression(GZIP_COMPRESSION) //
169 .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) //
170 .fileFormatVersion(FILE_FORMAT_VERSION) //
171 .changeIdentifier(CHANGE_IDENTIFIER) //
176 static void setUpConfiguration() {
177 when(appConfigMock.getCertificateConfiguration()).thenReturn(certificateConfigMock);
178 appConfigMock.collectedFilesPath = DATAFILE_TMPDIR;
179 certificateConfigMock.keyPasswordPath = CERTIFICATE_KEY_PASSWORD_PATH;
180 certificateConfigMock.trustedCa = TRUSTED_CA_PATH;
181 certificateConfigMock.trustedCaPasswordPath = TRUSTED_CA_PASSWORD_PATH;
186 counters = new Counters();
190 public void whenFtpesFile_returnCorrectResponse() throws Exception {
191 FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
192 doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
194 FileData fileData = createFileData(FTPES_LOCATION_NO_PORT);
196 FilePublishInformation expectedfilePublishInformation =
197 createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
199 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
200 .expectNext(expectedfilePublishInformation) //
203 verify(ftpesClientMock, times(1)).open();
204 verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
205 verify(ftpesClientMock, times(1)).close();
206 verifyNoMoreInteractions(ftpesClientMock);
208 assertEquals(1, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
209 assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0");
210 assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
214 public void whenSftpFile_returnCorrectResponse() throws Exception {
215 FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
216 doReturn(sftpClientMock).when(collectorUndetTest).createSftpClient(any());
218 FileData fileData = createFileData(SFTP_LOCATION_NO_PORT);
219 FilePublishInformation expectedfilePublishInformation =
220 createExpectedFilePublishInformation(SFTP_LOCATION_NO_PORT);
222 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
223 .expectNext(expectedfilePublishInformation) //
226 // The same again, but with port
227 fileData = createFileData(SFTP_LOCATION);
228 expectedfilePublishInformation = createExpectedFilePublishInformation(SFTP_LOCATION);
230 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
231 .expectNext(expectedfilePublishInformation) //
234 verify(sftpClientMock, times(2)).open();
235 verify(sftpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
236 verify(sftpClientMock, times(2)).close();
237 verifyNoMoreInteractions(sftpClientMock);
239 assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 2");
243 public void whenHttpFile_returnCorrectResponse() throws Exception {
244 FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
245 doReturn(dfcHttpClientMock).when(collectorUndetTest).createHttpClient(any());
247 FileData fileData = createFileData(HTTP_LOCATION_NO_PORT);
249 FilePublishInformation expectedfilePublishInformation =
250 createExpectedFilePublishInformation(HTTP_LOCATION_NO_PORT);
252 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
253 .expectNext(expectedfilePublishInformation) //
256 // The same again, but with port
257 fileData = createFileData(HTTP_LOCATION);
258 expectedfilePublishInformation = createExpectedFilePublishInformation(HTTP_LOCATION);
260 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
261 .expectNext(expectedfilePublishInformation) //
264 verify(dfcHttpClientMock, times(2)).open();
265 verify(dfcHttpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
266 verify(dfcHttpClientMock, times(2)).close();
267 verifyNoMoreInteractions(dfcHttpClientMock);
269 assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
270 assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0");
271 assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
275 public void whenHttpsFile_returnCorrectResponse() throws Exception {
276 FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
277 doReturn(dfcHttpsClientMock).when(collectorUndetTest).createHttpsClient(any());
279 FileData fileData = createFileData(HTTPS_LOCATION_NO_PORT);
281 FilePublishInformation expectedfilePublishInformation =
282 createExpectedFilePublishInformation(HTTPS_LOCATION_NO_PORT);
284 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
285 .expectNext(expectedfilePublishInformation) //
288 // The same again, but with port
289 fileData = createFileData(HTTPS_LOCATION);
290 expectedfilePublishInformation = createExpectedFilePublishInformation(HTTPS_LOCATION);
292 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
293 .expectNext(expectedfilePublishInformation) //
296 verify(dfcHttpsClientMock, times(2)).open();
297 verify(dfcHttpsClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
298 verify(dfcHttpsClientMock, times(2)).close();
299 verifyNoMoreInteractions(dfcHttpsClientMock);
301 assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
302 assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0");
303 assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
307 public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception {
308 FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
309 doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
311 FileData fileData = createFileData(FTPES_LOCATION);
312 doThrow(new DatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
313 .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
315 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
316 .expectErrorMessage("Retries exhausted: 3/3") //
319 verify(ftpesClientMock, times(4)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
321 assertEquals(0, counters.getNoOfCollectedFiles(), "collectedFiles should have been 0");
322 assertEquals(4, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 4");
323 assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
327 public void whenFtpesFileAlwaysFail_failWithoutRetry() throws Exception {
328 FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
329 doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
331 FileData fileData = createFileData(FTPES_LOCATION);
332 doThrow(new NonRetryableDatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
333 .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
335 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
336 .expectErrorMessage("Non retryable file transfer failure") //
339 verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
341 assertEquals(0, counters.getNoOfCollectedFiles(), "collectedFiles should have been 0");
342 assertEquals(1, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 1");
343 assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
347 public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() throws Exception {
348 FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
349 doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
350 doThrow(new DatafileTaskException("Unable to collect file.")).doNothing().when(ftpesClientMock)
351 .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
353 FilePublishInformation expectedfilePublishInformation =
354 createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
356 FileData fileData = createFileData(FTPES_LOCATION_NO_PORT);
358 StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
359 .expectNext(expectedfilePublishInformation) //
362 verify(ftpesClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
364 assertEquals(1, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
365 assertEquals(1, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 1");
366 assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");