datafilecollector/src/test/java/org/oran/datafile/tasks/FileCollectorTest.java

   1 /*-
   2  * ============LICENSE_START======================================================================
   3  * Copyright (C) 2018-2023 Nordix Foundation. All rights reserved.
   4  * Copyright (C) 2020-2022 Nokia. All rights reserved.
   5  * ===============================================================================================
   6  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
   7  * in compliance with the License. You may obtain a copy of the License at
   8  *
   9  * http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software distributed under the License
  12  * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
  13  * or implied. See the License for the specific language governing permissions and limitations under
  14  * the License.
  15  * ============LICENSE_END========================================================================
  16  */
  17
  18 package org.oran.datafile.tasks;
  19
  20 import static org.junit.jupiter.api.Assertions.assertEquals;
  21 import static org.mockito.ArgumentMatchers.any;
  22 import static org.mockito.Mockito.doReturn;
  23 import static org.mockito.Mockito.doThrow;
  24 import static org.mockito.Mockito.mock;
  25 import static org.mockito.Mockito.spy;
  26 import static org.mockito.Mockito.times;
  27 import static org.mockito.Mockito.verify;
  28 import static org.mockito.Mockito.verifyNoMoreInteractions;
  29 import static org.mockito.Mockito.when;
  30
  31 import java.nio.file.Path;
  32 import java.nio.file.Paths;
  33 import java.time.Duration;
  34 import java.util.ArrayList;
  35 import java.util.List;
  36
  37 import org.junit.jupiter.api.BeforeAll;
  38 import org.junit.jupiter.api.BeforeEach;
  39 import org.junit.jupiter.api.Test;
  40 import org.oran.datafile.configuration.AppConfig;
  41 import org.oran.datafile.configuration.CertificateConfig;
  42 import org.oran.datafile.exceptions.DatafileTaskException;
  43 import org.oran.datafile.exceptions.NonRetryableDatafileTaskException;
  44 import org.oran.datafile.ftp.FtpesClient;
  45 import org.oran.datafile.ftp.SftpClient;
  46 import org.oran.datafile.http.DfcHttpClient;
  47 import org.oran.datafile.http.DfcHttpsClient;
  48 import org.oran.datafile.model.Counters;
  49 import org.oran.datafile.model.FileData;
  50 import org.oran.datafile.model.FilePublishInformation;
  51 import org.oran.datafile.model.FileReadyMessage;
  52 import org.oran.datafile.oauth2.SecurityContext;
  53 import reactor.test.StepVerifier;
  54
  55 public class FileCollectorTest {
  56
  57     final static String DATAFILE_TMPDIR = "/tmp/onap_datafile/";
  58     private static final String PRODUCT_NAME = "NrRadio";
  59     private static final String VENDOR_NAME = "Ericsson";
  60     private static final int LAST_EPOCH_MICROSEC = 87457457;
  61     private static final String SOURCE_NAME = "oteNB5309";
  62     private static final int START_EPOCH_MICROSEC = 874575764;
  63     private static final String TIME_ZONE_OFFSET = "UTC+05:00";
  64     private static final String FTPES_SCHEME = "ftpes://";
  65     private static final String SFTP_SCHEME = "sftp://";
  66     private static final String HTTP_SCHEME = "http://";
  67     private static final String HTTPS_SCHEME = "https://";
  68     private static final String SERVER_ADDRESS = "192.168.0.101";
  69     private static final int PORT_22 = 22;
  70     private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz";
  71     private static final Path LOCAL_FILE_LOCATION = Paths.get(DATAFILE_TMPDIR, SOURCE_NAME, PM_FILE_NAME);
  72     private static final String REMOTE_FILE_LOCATION = "/ftp/rop/" + PM_FILE_NAME;
  73     private static final String USER = "usr";
  74     private static final String PWD = "pwd";
  75     private static final String FTPES_LOCATION =
  76         FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  77
  78     private static final String FTPES_LOCATION_NO_PORT =
  79         FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  80     private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  81     private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  82
  83     private static final String HTTP_LOCATION =
  84         HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  85     private static final String HTTP_LOCATION_NO_PORT =
  86         HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  87     private static final String HTTPS_LOCATION =
  88         HTTPS_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  89     private static final String HTTPS_LOCATION_NO_PORT =
  90         HTTPS_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  91
  92     private static final String GZIP_COMPRESSION = "gzip";
  93     private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
  94     private static final String FILE_FORMAT_VERSION = "V10";
  95     private static final String CERTIFICATE_KEY_PASSWORD_PATH = "certificateKeyPassword";
  96     private static final String TRUSTED_CA_PATH = "trustedCAPath";
  97     private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword";
  98     private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES";
  99     private static final String FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
 100     private static final String CHANGE_TYPE = "FileReady";
 101
 102     private static AppConfig appConfigMock = mock(AppConfig.class);
 103     private static CertificateConfig certificateConfigMock = mock(CertificateConfig.class);
 104
 105     private FtpesClient ftpesClientMock = mock(FtpesClient.class);
 106
 107     private SftpClient sftpClientMock = mock(SftpClient.class);
 108
 109     private DfcHttpClient dfcHttpClientMock = mock(DfcHttpClient.class);
 110     private DfcHttpsClient dfcHttpsClientMock = mock(DfcHttpsClient.class);
 111
 112     private Counters counters;
 113
 114     static final SecurityContext securityContext = new SecurityContext("");
 115
 116     FileReadyMessage.Event event(String location) {
 117         FileReadyMessage.MessageMetaData messageMetaData = FileReadyMessage.MessageMetaData.builder() //
 118             .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
 119             .sourceName(SOURCE_NAME) //
 120             .startEpochMicrosec(START_EPOCH_MICROSEC) //
 121             .timeZoneOffset(TIME_ZONE_OFFSET) //
 122             .changeIdentifier(CHANGE_IDENTIFIER) //
 123             .eventName("Noti_NrRadio-Ericsson_FileReady").build();
 124
 125         FileReadyMessage.FileInfo fileInfo = FileReadyMessage.FileInfo //
 126             .builder() //
 127             .fileFormatType(FILE_FORMAT_TYPE) //
 128             .location(location) //
 129             .fileFormatVersion(FILE_FORMAT_VERSION) //
 130             .compression(GZIP_COMPRESSION) //
 131             .build();
 132
 133         FileReadyMessage.ArrayOfNamedHashMap arrayOfNamedHashMap = FileReadyMessage.ArrayOfNamedHashMap //
 134             .builder().name(PM_FILE_NAME) //
 135             .hashMap(fileInfo).build();
 136
 137         List<FileReadyMessage.ArrayOfNamedHashMap> arrayOfNamedHashMapList = new ArrayList<>();
 138         arrayOfNamedHashMapList.add(arrayOfNamedHashMap);
 139
 140         FileReadyMessage.NotificationFields notificationFields = FileReadyMessage.NotificationFields //
 141             .builder().notificationFieldsVersion("notificationFieldsVersion") //
 142             .changeType(CHANGE_TYPE).changeIdentifier(CHANGE_IDENTIFIER) //
 143             .arrayOfNamedHashMap(arrayOfNamedHashMapList) //
 144             .build();
 145
 146         return FileReadyMessage.Event.builder() //
 147             .commonEventHeader(messageMetaData) //
 148             .notificationFields(notificationFields).build();
 149     }
 150
 151     private FileReadyMessage fileReadyMessage(String location) {
 152         FileReadyMessage message = FileReadyMessage.builder() //
 153             .event(event(location)) //
 154             .build();
 155         return message;
 156     }
 157
 158     private FileData createFileData(String location) {
 159         return FileData.createFileData(fileReadyMessage(location)).iterator().next();
 160     }
 161
 162     private FilePublishInformation createExpectedFilePublishInformation(String location) {
 163         return FilePublishInformation.builder() //
 164             .productName(PRODUCT_NAME) //
 165             .vendorName(VENDOR_NAME) //
 166             .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
 167             .sourceName(SOURCE_NAME) //
 168             .startEpochMicrosec(START_EPOCH_MICROSEC) //
 169             .timeZoneOffset(TIME_ZONE_OFFSET) //
 170             .name(SOURCE_NAME + "/" + PM_FILE_NAME) //
 171             .compression(GZIP_COMPRESSION) //
 172             .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) //
 173             .fileFormatVersion(FILE_FORMAT_VERSION) //
 174             .changeIdentifier(CHANGE_IDENTIFIER) //
 175             .build();
 176     }
 177
 178     @BeforeAll
 179     static void setUpConfiguration() {
 180         when(appConfigMock.getCertificateConfiguration()).thenReturn(certificateConfigMock);
 181         when(appConfigMock.getCollectedFilesPath()).thenReturn(DATAFILE_TMPDIR);
 182         certificateConfigMock.keyPasswordPath = CERTIFICATE_KEY_PASSWORD_PATH;
 183         certificateConfigMock.trustedCa = TRUSTED_CA_PATH;
 184         certificateConfigMock.trustedCaPasswordPath = TRUSTED_CA_PASSWORD_PATH;
 185     }
 186
 187     @BeforeEach
 188     void setUpTest() {
 189         counters = new Counters();
 190     }
 191
 192     @Test
 193     public void whenFtpesFile_returnCorrectResponse() throws Exception {
 194         FileCollector collectorUndetTest = spy(new FileCollector(securityContext, appConfigMock, counters));
 195         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 196
 197         FileData fileData = createFileData(FTPES_LOCATION_NO_PORT);
 198
 199         FilePublishInformation expectedfilePublishInformation =
 200             createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
 201
 202         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 203             .expectNext(expectedfilePublishInformation) //
 204             .verifyComplete();
 205
 206         verify(ftpesClientMock, times(1)).open();
 207         verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 208         verify(ftpesClientMock, times(1)).close();
 209         verifyNoMoreInteractions(ftpesClientMock);
 210
 211         assertEquals(1, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
 212         assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0");
 213         assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
 214     }
 215
 216     @Test
 217     public void whenSftpFile_returnCorrectResponse() throws Exception {
 218         FileCollector collectorUndetTest = spy(new FileCollector(securityContext, appConfigMock, counters));
 219         doReturn(sftpClientMock).when(collectorUndetTest).createSftpClient(any());
 220
 221         FileData fileData = createFileData(SFTP_LOCATION_NO_PORT);
 222         FilePublishInformation expectedfilePublishInformation =
 223             createExpectedFilePublishInformation(SFTP_LOCATION_NO_PORT);
 224
 225         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 226             .expectNext(expectedfilePublishInformation) //
 227             .verifyComplete();
 228
 229         // The same again, but with port
 230         fileData = createFileData(SFTP_LOCATION);
 231         expectedfilePublishInformation = createExpectedFilePublishInformation(SFTP_LOCATION);
 232
 233         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 234             .expectNext(expectedfilePublishInformation) //
 235             .verifyComplete();
 236
 237         verify(sftpClientMock, times(2)).open();
 238         verify(sftpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 239         verify(sftpClientMock, times(2)).close();
 240         verifyNoMoreInteractions(sftpClientMock);
 241
 242         assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 2");
 243     }
 244
 245     @Test
 246     public void whenHttpFile_returnCorrectResponse() throws Exception {
 247         FileCollector collectorUndetTest = spy(new FileCollector(securityContext, appConfigMock, counters));
 248         doReturn(dfcHttpClientMock).when(collectorUndetTest).createHttpClient(any());
 249
 250         FileData fileData = createFileData(HTTP_LOCATION_NO_PORT);
 251
 252         FilePublishInformation expectedfilePublishInformation =
 253             createExpectedFilePublishInformation(HTTP_LOCATION_NO_PORT);
 254
 255         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 256             .expectNext(expectedfilePublishInformation) //
 257             .verifyComplete();
 258
 259         // The same again, but with port
 260         fileData = createFileData(HTTP_LOCATION);
 261         expectedfilePublishInformation = createExpectedFilePublishInformation(HTTP_LOCATION);
 262
 263         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 264             .expectNext(expectedfilePublishInformation) //
 265             .verifyComplete();
 266
 267         verify(dfcHttpClientMock, times(2)).open();
 268         verify(dfcHttpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 269         verify(dfcHttpClientMock, times(2)).close();
 270         verifyNoMoreInteractions(dfcHttpClientMock);
 271
 272         assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
 273         assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0");
 274         assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
 275     }
 276
 277     @Test
 278     public void whenHttpsFile_returnCorrectResponse() throws Exception {
 279         FileCollector collectorUndetTest = spy(new FileCollector(securityContext, appConfigMock, counters));
 280         doReturn(dfcHttpsClientMock).when(collectorUndetTest).createHttpsClient(any());
 281
 282         FileData fileData = createFileData(HTTPS_LOCATION_NO_PORT);
 283
 284         FilePublishInformation expectedfilePublishInformation =
 285             createExpectedFilePublishInformation(HTTPS_LOCATION_NO_PORT);
 286
 287         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 288             .expectNext(expectedfilePublishInformation) //
 289             .verifyComplete();
 290
 291         // The same again, but with port
 292         fileData = createFileData(HTTPS_LOCATION);
 293         expectedfilePublishInformation = createExpectedFilePublishInformation(HTTPS_LOCATION);
 294
 295         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 296             .expectNext(expectedfilePublishInformation) //
 297             .verifyComplete();
 298
 299         verify(dfcHttpsClientMock, times(2)).open();
 300         verify(dfcHttpsClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 301         verify(dfcHttpsClientMock, times(2)).close();
 302         verifyNoMoreInteractions(dfcHttpsClientMock);
 303
 304         assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
 305         assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0");
 306         assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
 307     }
 308
 309     @Test
 310     public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception {
 311         FileCollector collectorUndetTest = spy(new FileCollector(securityContext, appConfigMock, counters));
 312         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 313
 314         FileData fileData = createFileData(FTPES_LOCATION);
 315         doThrow(new DatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
 316             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 317
 318         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 319             .expectErrorMessage("Retries exhausted: 3/3") //
 320             .verify();
 321
 322         verify(ftpesClientMock, times(4)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 323
 324         assertEquals(0, counters.getNoOfCollectedFiles(), "collectedFiles should have been 0");
 325         assertEquals(4, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 4");
 326         assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
 327     }
 328
 329     @Test
 330     public void whenFtpesFileAlwaysFail_failWithoutRetry() throws Exception {
 331         FileCollector collectorUndetTest = spy(new FileCollector(securityContext, appConfigMock, counters));
 332         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 333
 334         FileData fileData = createFileData(FTPES_LOCATION);
 335         doThrow(new NonRetryableDatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
 336             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 337
 338         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 339             .expectErrorMessage("Non retryable file transfer failure") //
 340             .verify();
 341
 342         verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 343
 344         assertEquals(0, counters.getNoOfCollectedFiles(), "collectedFiles should have been 0");
 345         assertEquals(1, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 1");
 346         assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
 347     }
 348
 349     @Test
 350     public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() throws Exception {
 351         FileCollector collectorUndetTest = spy(new FileCollector(securityContext, appConfigMock, counters));
 352         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 353         doThrow(new DatafileTaskException("Unable to collect file.")).doNothing().when(ftpesClientMock)
 354             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 355
 356         FilePublishInformation expectedfilePublishInformation =
 357             createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
 358
 359         FileData fileData = createFileData(FTPES_LOCATION_NO_PORT);
 360
 361         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0)))
 362             .expectNext(expectedfilePublishInformation) //
 363             .verifyComplete();
 364
 365         verify(ftpesClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 366
 367         assertEquals(1, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1");
 368         assertEquals(1, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 1");
 369         assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0");
 370     }
 371 }