From a363dc5ca8922b41768aad60f418647ea1e4e5fe Mon Sep 17 00:00:00 2001 From: PatrikBuhr Date: Fri, 3 Mar 2023 14:02:50 +0100 Subject: [PATCH] Creating datafile Datafile collector, wich is moved from the prototype repo. Signed-off-by: PatrikBuhr Issue-ID: NONRTRIC-852 Change-Id: I1b91a51e328dc8cd11c14b290fe7296ed165ddf4 --- datafilecollector/.gitignore | 54 +++ datafilecollector/Dockerfile | 51 +++ datafilecollector/LICENSE.txt | 36 ++ datafilecollector/README.md | 37 ++ datafilecollector/config/README | 43 +++ datafilecollector/config/application.yaml | 56 +++ datafilecollector/config/ftps_keystore.p12 | Bin 0 -> 2857 bytes datafilecollector/config/ftps_keystore.pass | 1 + datafilecollector/config/keystore.jks | Bin 0 -> 5269 bytes datafilecollector/config/truststore.jks | Bin 0 -> 3869 bytes datafilecollector/config/truststore.pass | 1 + datafilecollector/onap-java-formatter.xml | 296 +++++++++++++++ datafilecollector/pom.xml | 334 ++++++++++++++++ .../onap/dcaegen2/collectors/datafile/MainApp.java | 47 +++ .../datafile/commons/FileCollectClient.java | 32 ++ .../datafile/commons/FileServerData.java | 51 +++ .../collectors/datafile/commons/Scheme.java | 70 ++++ .../collectors/datafile/commons/SecurityUtil.java | 52 +++ .../datafile/configuration/AppConfig.java | 126 +++++++ .../datafile/configuration/CertificateConfig.java | 36 ++ .../datafile/configuration/SftpConfig.java | 31 ++ .../datafile/configuration/SwaggerConfig.java | 42 +++ .../datafile/controllers/StatusController.java | 99 +++++ .../collectors/datafile/datastore/DataStore.java | 57 +++ .../collectors/datafile/datastore/FileStore.java | 160 ++++++++ .../datafile/datastore/S3ObjectStore.java | 313 +++++++++++++++ .../datafile/exceptions/DatafileTaskException.java | 33 ++ .../exceptions/EnvironmentLoaderException.java | 33 ++ .../NonRetryableDatafileTaskException.java | 33 ++ .../collectors/datafile/ftp/FtpesClient.java | 230 +++++++++++ .../collectors/datafile/ftp/SftpClient.java | 151 ++++++++ .../datafile/ftp/SftpClientSettings.java | 60 +++ .../collectors/datafile/http/DfcHttpClient.java | 179 +++++++++ .../collectors/datafile/http/DfcHttpsClient.java | 182 +++++++++ .../http/HttpAsyncClientBuilderWrapper.java | 59 +++ .../http/HttpsClientConnectionManagerUtil.java | 138 +++++++ .../collectors/datafile/model/Counters.java | 145 +++++++ .../collectors/datafile/model/FileData.java | 162 ++++++++ .../datafile/model/FilePublishInformation.java | 52 +++ .../datafile/model/FileReadyMessage.java | 111 ++++++ .../collectors/datafile/service/HttpUtils.java | 229 +++++++++++ .../datafile/tasks/CollectAndReportFiles.java | 301 +++++++++++++++ .../collectors/datafile/tasks/FileCollector.java | 187 +++++++++ .../datafile/tasks/KafkaTopicListener.java | 106 ++++++ .../dcaegen2/collectors/datafile/MockDatafile.java | 341 +++++++++++++++++ .../datafile/controllers/StatusControllerTest.java | 73 ++++ .../collectors/datafile/ftp/FtpesClientTest.java | 238 ++++++++++++ .../datafile/ftp/SftpClientSettingsTest.java | 67 ++++ .../collectors/datafile/ftp/SftpClientTest.java | 237 ++++++++++++ .../datafile/http/DfcHttpClientTest.java | 159 ++++++++ .../datafile/http/DfcHttpsClientTest.java | 178 +++++++++ .../datafile/http/HttpClientResponseHelper.java | 419 +++++++++++++++++++++ .../http/HttpsClientConnectionManagerUtilTest.java | 54 +++ .../collectors/datafile/scheme/SchemeTest.java | 48 +++ .../collectors/datafile/service/HttpUtilsTest.java | 159 ++++++++ .../datafile/tasks/FileCollectorTest.java | 368 ++++++++++++++++++ .../collectors/datafile/utils/JsonMessage.java | 262 +++++++++++++ .../collectors/datafile/utils/LoggingUtils.java | 56 +++ datafilecollector/src/test/resources/cert.jks | Bin 0 -> 4888 bytes datafilecollector/src/test/resources/dfc.jks | Bin 0 -> 2151 bytes datafilecollector/src/test/resources/dfc.jks.pass | 1 + datafilecollector/src/test/resources/ftp.jks.pass | 1 + datafilecollector/src/test/resources/jks.pass | 1 + datafilecollector/src/test/resources/keystore.p12 | Bin 0 -> 2857 bytes datafilecollector/src/test/resources/keystore.pass | 1 + datafilecollector/src/test/resources/trust.jks | Bin 0 -> 1413 bytes datafilecollector/src/test/resources/trust.pass | 1 + 67 files changed, 7080 insertions(+) create mode 100644 datafilecollector/.gitignore create mode 100755 datafilecollector/Dockerfile create mode 100644 datafilecollector/LICENSE.txt create mode 100644 datafilecollector/README.md create mode 100644 datafilecollector/config/README create mode 100644 datafilecollector/config/application.yaml create mode 100755 datafilecollector/config/ftps_keystore.p12 create mode 100755 datafilecollector/config/ftps_keystore.pass create mode 100644 datafilecollector/config/keystore.jks create mode 100644 datafilecollector/config/truststore.jks create mode 100755 datafilecollector/config/truststore.pass create mode 100644 datafilecollector/onap-java-formatter.xml create mode 100644 datafilecollector/pom.xml create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/MainApp.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileCollectClient.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SftpConfig.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SwaggerConfig.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusController.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/DataStore.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/FileStore.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/S3ObjectStore.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/DatafileTaskException.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/EnvironmentLoaderException.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/NonRetryableDatafileTaskException.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettings.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpAsyncClientBuilderWrapper.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FilePublishInformation.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileReadyMessage.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/CollectAndReportFiles.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java create mode 100644 datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/KafkaTopicListener.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/MockDatafile.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusControllerTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettingsTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/JsonMessage.java create mode 100644 datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/LoggingUtils.java create mode 100755 datafilecollector/src/test/resources/cert.jks create mode 100644 datafilecollector/src/test/resources/dfc.jks create mode 100644 datafilecollector/src/test/resources/dfc.jks.pass create mode 100644 datafilecollector/src/test/resources/ftp.jks.pass create mode 100755 datafilecollector/src/test/resources/jks.pass create mode 100644 datafilecollector/src/test/resources/keystore.p12 create mode 100644 datafilecollector/src/test/resources/keystore.pass create mode 100755 datafilecollector/src/test/resources/trust.jks create mode 100755 datafilecollector/src/test/resources/trust.pass diff --git a/datafilecollector/.gitignore b/datafilecollector/.gitignore new file mode 100644 index 0000000..9ec364a --- /dev/null +++ b/datafilecollector/.gitignore @@ -0,0 +1,54 @@ +# Compiled class file +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# Intellij IDE +.idea +*.iml + +# Eclipse IDE +.project +.classpath +.settings +bin + +# Maven +target +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +.mvn/wrapper/maven-wrapper.jar + +# CheckStyle files +.checkstyle + +opt/ + +# Visual Studio Code +.factorypath diff --git a/datafilecollector/Dockerfile b/datafilecollector/Dockerfile new file mode 100755 index 0000000..cee8d75 --- /dev/null +++ b/datafilecollector/Dockerfile @@ -0,0 +1,51 @@ +# +# ============LICENSE_START======================================================= +# Copyright (C) 2023 Nordix Foundation. +# Copyright (C) 2020 Nokia. +# Copyright (C) 2021 Samsung Electronics. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +# +FROM openjdk:17-jdk-slim + +EXPOSE 8100 8433 + +ARG user=datafile +ARG group=datafile + +USER root +WORKDIR /opt/app/datafile + +ADD /config/application.yaml /opt/app/datafile/config/ +ADD /config/ftps_keystore.pass /opt/app/datafile/config/ +ADD /config/ftps_keystore.p12 /opt/app/datafile/config/ +ADD /config/keystore.jks /opt/app/datafile/config/ +ADD /config/truststore.jks /opt/app/datafile/config/ +ADD /config/truststore.pass /opt/app/datafile/config/ + + + + +RUN mkdir -p /var/log/ONAP /opt/app/datafile/etc/cert/ && \ + addgroup $group && adduser --system --disabled-password --no-create-home --ingroup $group $user && \ + chown -R $user:$group /var/log/ONAP /opt/app/datafile/config && \ + chmod -R u+rw /opt/app/datafile/config/ + + +USER $user + +COPY --chown=$user:$group /target/datafile-app-server.jar /opt/app/datafile/ +ENTRYPOINT ["java", "-jar", "/opt/app/datafile/datafile-app-server.jar"] diff --git a/datafilecollector/LICENSE.txt b/datafilecollector/LICENSE.txt new file mode 100644 index 0000000..581378e --- /dev/null +++ b/datafilecollector/LICENSE.txt @@ -0,0 +1,36 @@ +/* +* ============LICENSE_START========================================== +* Copyright (C) 2018-2023 Nordix Foundation. All rights reserved. +* Copyright (c) 2018 NOKIA Intellectual Property. All rights reserved. +* =================================================================== +* +* Unless otherwise specified, all software contained herein is licensed +* under the Apache License, Version 2.0 (the "License"); +* you may not use this software except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* +* +* Unless otherwise specified, all documentation contained herein is licensed +* under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +* you may not use this documentation except in compliance with the License. +* You may obtain a copy of the License at +* +* https://creativecommons.org/licenses/by/4.0/ +* +* Unless required by applicable law or agreed to in writing, documentation +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* ============LICENSE_END============================================ +*/ diff --git a/datafilecollector/README.md b/datafilecollector/README.md new file mode 100644 index 0000000..785a46d --- /dev/null +++ b/datafilecollector/README.md @@ -0,0 +1,37 @@ +# DFC (DataFile Collector) + +Datafile Collector is responsible for collecting PM counter files from traffical functions. +The files are stored in a persistent volume or in an S3 object store. + +The origin is from ONAP. This variant uses Kafka and S3 object store and does not use the Dmaap. + +## Introduction + +DFC is delivered as one **Docker container** which hosts application server and can be started by `docker-compose`. + +## Compiling DFC + +Whole project (top level of DFC directory) and each module (sub module directory) can be compiled using +`mvn clean install` command. + +## Build image +``` +mvn install docker:build +``` + +## Main API Endpoints + +Running with dev-mode of DFC + +- **Heartbeat**: http://:8100/**heartbeat** or https://:8443/**heartbeat** + +- **Start DFC**: http://:8100/**start** or https://:8433/**start** + +- **Stop DFC**: http://:8100/**stopDatafile** or https://:8433/**stopDatafile** + + + +## License + +Copyright (C) 2018-2019 NOKIA Intellectual Property, 2018-2023 Nordix Foundation. All rights reserved. +[License](http://www.apache.org/licenses/LICENSE-2.0) diff --git a/datafilecollector/config/README b/datafilecollector/config/README new file mode 100644 index 0000000..cfde02e --- /dev/null +++ b/datafilecollector/config/README @@ -0,0 +1,43 @@ +The keystore.jks and truststore.jks files are created by using the following commands (note that this is an example): + +1) Create a CA certificate and a private key: + +openssl genrsa -des3 -out CA-key.pem 2048 +openssl req -new -key CA-key.pem -x509 -days 3600 -out CA-cert.pem + +2) Create a keystore with a private key entry that is signed by the CA: + +Note: your name must be "localhost" + +keytool -genkeypair -alias policy_agent -keyalg RSA -keysize 2048 -keystore keystore.jks -validity 3650 -storepass policy_agent +keytool -certreq -alias policy_agent -file request.csr -keystore keystore.jks -ext san=dns:your.domain.com -storepass policy_agent +openssl x509 -req -days 3650 -in request.csr -CA CA-cert.pem -CAkey CA-key.pem -CAcreateserial -out ca_signed-cert.pem +keytool -importcert -alias ca_cert -file CA-cert.pem -keystore keystore.jks -trustcacerts -storepass policy_agent +keytool -importcert -alias policy_agent -file ca_signed-cert.pem -keystore keystore.jks -trustcacerts -storepass policy_agent + + +3) Create a trust store containing the CA cert (to trust all certs signed by the CA): + +keytool -genkeypair -alias not_used -keyalg RSA -keysize 2048 -keystore truststore.jks -validity 3650 -storepass policy_agent +keytool -importcert -alias ca_cert -file CA-cert.pem -keystore truststore.jks -trustcacerts -storepass policy_agent + + +4) Command for listing of the contents of jks files, examples: +keytool -list -v -keystore keystore.jks -storepass policy_agent +keytool -list -v -keystore truststore.jks -storepass policy_agent + +## License + +Copyright (C) 2022 Nordix Foundation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/datafilecollector/config/application.yaml b/datafilecollector/config/application.yaml new file mode 100644 index 0000000..71f3172 --- /dev/null +++ b/datafilecollector/config/application.yaml @@ -0,0 +1,56 @@ +spring: + profiles: + active: prod +management: + endpoints: + web: + exposure: + include: "loggers,logfile,health,info,metrics" +server: + port: 8433 + ssl: + key-store-type: JKS + key-store-password: policy_agent + key-store: config/keystore.jks + key-password: policy_agent + key-alias: policy_agent +logging: + level: + ROOT: WARN + org.onap: WARN + org.springframework: WARN + org.springframework.data: WARN + org.springframework.web.reactive.function.client.ExchangeFunctions: WARN + org.onap.dcaegen2.collectors.datafile: INFO + + file: + name: /var/log/ONAP/application.log +app: + filepath: config/datafile_endpoints_test.json + collected-files-path: "/tmp/onap_datafile/" + # KAFKA boostrap servers. This is only needed if there are Information Types that uses a kafkaInputTopic + # several redundant boostrap servers can be specified, separated by a comma ','. + kafka: + bootstrap-servers: localhost:9092 + # output topic + collected-file-topic: collected-file + client-id: datafile-1 + # input topic + file-ready-event-topic: file-ready + sftp: + known-hosts-file-path: + strict-host-key-checking: false + ssl: + key-store-password-file: /opt/app/datafile/config/ftps_keystore.pass + key-store: /opt/app/datafile/config/ftps_keystore.p12 + trust-store-password-file: /opt/app/datafile/config/truststore.pass + trust-store: /opt/app/datafile/config/truststore.jks + s3: + endpointOverride: + accessKeyId: + secretAccessKey: + bucket: + locksBucket: +springdoc: + show-actuator: true + swagger-ui.disable-swagger-default-url: true \ No newline at end of file diff --git a/datafilecollector/config/ftps_keystore.p12 b/datafilecollector/config/ftps_keystore.p12 new file mode 100755 index 0000000000000000000000000000000000000000..b847707c846ffdb1e13a61840d51728b88c82753 GIT binary patch literal 2857 zcmY+Ec{mh|7RF~7gQ1AAXD1Qv7-fwtW6d&Zh*8;hW8X5?$u0(262_82+4nMJ-_?jv zWUH|>LRnIlT=zcre&6%`@jmA~@B5zf=MRQs5~l&u!f=oUFr!qQe%!%1AU&`M2O)!S zkkK=H5Qd}k|3^gU2*T0Xp4oTLWC>>a-xV_iNK=FZ9l>y*1DGTj@_+m6IUfj`@L+`Q zf?+5o}BLS^j3gQOeG zzc>&2)9k$Gzlii|SUz>Ev@4n4U{LE}igQ}@qjIIR4NzzM*+lX%Vap>9l&J_pyqNli zgOUhfb)xp!bwF*dE99i{70NC=yKZS84J(uGs2TqO`!L zER6%7yTCJ&iTFJtWGCId`U9Y95vQu3Hev)DzU-U_pwZq1;Rc1x$sI%lhTcx zdWPg%*Q%v?IwrLX(zxk!7Z9D|W*+><;yDIC{f><2kfNjf5h{m_uX*s2?seaEEu>^Q zDsEIhIlXT4cDw9PSXKkIj<0U`*s_u^F;RnQ+Fl=>a67(k(Q?CSk3Fh-Qz@_N8(KG@ zaP-u@@sB23RO~v3&m*1|>DrjYn|;q!)oSkCRsV47?R5;x-NYvYj8AeUqGPW32{|M1TN(>Htav#Z@Wnkdp>=h65lzMX!cSh z0YSy~J|^5S96@}2u~Q}59*v)H;WvLzZ>P;mtd~kgcHK7*pMo0nyFzFoiij$q0@BwJ zJ71gP%{ed@HT^OQ8FnK3)RQA<{9L{MSW4kI+$!`pvyKo7-e=NB4w5O=BY{)1A+|Lg zRakkH=%n{L-zMi3#X^?_;Bpb11Lh ztdIJv#M!JGi{BfP@ABixqnss9hQ;Nep>m%ZE7vS2>0B1-dLOH&#W=%GNA0VKoFKN& zGM)q5`H*B6UP>~bRs7`p=)t-^%>s%erh1QO5ejpeQTD=ekNsInq6m*o&B$bVy1B4&1lj#LarL4`w*;cGzG9k+5{p@At! z#d(ssVS!gnXiU1gdJiL|&Q@U+$PL29WiR4;e8zgWxZ7(H`Tr8_*(2*ZU$`3lSo8r6O zQtGL*OxRMA=FGbv;~}eJm+4u9QBKt!n9Ean>s%T~zgXs>RXlljFAmU#20zL9 z`g;ALpw_7<)11sJx@NkHR1|U2MIlzzu?Skd^)`wXW`%U2^lD_!zkqnP{=fvrU2UiE zvv{Bvy37a#`8+EIgRTM_x%`sOv4z|59Iipu|yVPcb3;B z21tYI9#GWbh2;spuRbbz^MsYYa(Xtd4laH3U!>thyCZo|#b(sK>_)SVCo zult9srsL#=g0==^r?~2=7pR`U$%u@uzQ^OM8v%PzBm9SfzQY{6$(xh*B=@avmv$;D zXMSduRJcG1M7vCK>X(<-CQZ+?LEsS|FvF-gt3MRVxuw888NIt1SL&-0-ZK2Tdn|t{ zuryM^!5aZI!4-?a_J6lcg_RD^_9c4fY-4x)oPymwRP_DXmi2lEG9S8BVX;eii2IQv zI1CjO^sVA~fJY0jQ`Anq=$%^%ACN$M6B9wH>0LE;554d2EeHX@uf$a=IwwjGkCL_S zaiHv755q5~kHtow#P0P)yAPlUKeJmrzVbE7?|C)7x0p zCRk_9W*p|FvF6*vX_+qO-v$~LC8%&8aQAE1IKtyyc!)%MrvsVt3eJ854jA6GW&%LgiBN z(e!JAaE)Q1M!3(Yx6I6_`3q7{&;uyKuIR&G0&o t@U2-<*Qfdm71OoK4IrrQ_elK@K1!R4oOlZBY4$6gvh^_eMmvN>(Jeeq;zvY8kCf7=|)NpAR#CzDWG&ncZ0y8k(LsaxZljZ z^Zn;O-TSa-&;G5o_tTz*h=8YofM7%fTpfnR5v>~i8xM#9ER29lKqBBG|6l<`1oZ3w zlAx842W@hiia>ki2i59MQ}pk0ze*QNExfx z7jSxlKWZ-PoB>@80oHC29n ze1KVdh6*GCGRX&{S7_@FnzX;P$v$2WCul61=O16!I>$H4+0jNqi2xg zq0rw>jPI3ZL@?F4nOU(BM&thK+qAu`MGkZ@p;1 zd$T@;t7uI#jm_njppWOBat!wfcu3YR`o4q3F{p>ta_x`{MLc_sR+!7Z9r~7u&RGio z9zn#*G|(5B&RM^@)69c}mUfavW=5ZG_8%dls6@93P#kbZrid>y^Uu%(JZeS#a8^v} z*Sv(@l)W4pQu&h9R0Ie;=4zmh|2VO z&0(W;32Z;@Igr6!b8@2nyQ|G;8QCDUNah#q)8(gFdN3@k9up{;V3qf_LPb8)q04f8 ziAw4g&HV6JJC{kmP+X%q-g+ac=8`d9@#mfV_lEtx8j7>qo~3CVt(ZFE)GB&J%)3{P zb0YywEyj5>25w*d;9w?O4=!D@j`LW8a6ZBmvT>&|X#wriPeTbTM8S~_N$0KE<*VD5 zaBn!eN6}vBNzr&vC|Y@NGi_PIn(!G(vySao{wYJ%Q87HUq=4dquG23aIleQ#ydZQKBmj zU*UHyPoBhfWZ{IHD|i`-a%)8J^vs@%bMr99Xz*GsG+u9{f>x+_?6;Y;%Vb{ z1?QuBN&#>OyaKoaoB*}}e}EML32*?|0bT;U5%m8tlJdbJ#QIKdc65Bg!U%o@LKGp$ zCoF`B!1wsqgaIpzz_P>wP4b&NFr@GBQvLsQUGbSBhlZhId}D&&zYBlkj%AvWLMKy9c|n z24OzHnU$iHkgzP0-#Pz|!H>Yo{^z}d_%Xe?RWaM*p(*=IUGs!1oXhWrB;|&MB{#v5 zVH6{^ktO85htGA5wqqdiP+6vE+OY|Row#H6YahGAm$xiqc@1|7(`kx79ec5IXGoQk zb={)kb7cfIHAi0NqxooZYHWrHY4&?zMg3qW{rmO>ovy`(C&}3DOS3@&JEpI56W3Ce z3sycY#$osbj8qJ?GMqf!lpP4u@ygeh4JPh9_{#$yvz2_)Sn1Bwt{mCv@)8cNrsisq z8Mq62>?`hp42d{}V)dPK4Xhb1G|HY?JU9+M%@+tz_eahdjqnY!YY){*9bhy4 zkKTcOl(;y!YDSD=b~$M8-Qu)JlJSPNI#cc!=wi(nX!&oXo8qSTfbx8pYTbbbfFXXZ zN#!1=;!PC(*ALN4++-eOHMvDS3+RdrIs*!pAnw|~@?u9yI1bs24pJBA0*A;EiG~Tz zs^qsSk>0~?5r3-R=WqHk=ESmB=if~?usR2*splHLb$X)^I^EgFm*!z0vHSixFmLZM zGz2?*RiI(92^ZRPqPm+zpRu-e?^BiO(L3jBw&+g@MY^NhxC+-H#>0d}8Lj6e6}iL| zJ&a5d1jZI^^jD@Q?~YrQUj^9e|mmu@;&}^ zV`q$*2Bf@V7BEE(D6h{bl{QA~u~-PclDFJ-H!L|x2aH1;}DOU88)BG@iBgAd4tIU5G_a?bVip)_?{*`7-7unqkkWsmH z#)0Y!Ik?Pcz5uFp`EAD`_og6H;Fr-kw~5kt z;||{16&!uDu#;|rGM+&bRqlM&5!@7H+J8MkO}nb5m7h$c2mE~`uiclt9H3WC-IWTd zO64Bwr>)qy;k_b>AS@wrR1bVjXxRJM70apf`q`h9B_vgdjzHSiCDm3uPs@&GnZtB) zS|Q*Q+20LLl(ai8t-7xlb({ahnSB#ro++XAQ>_Y({Cmau$&gCuZ?Pw(e4EQsj-2j` zYet|>sqx_CP{J!#SQsUU%Vsr0DRZK5(u20$pd9=7a_20|h}ym7X|}|QPdhPbzJHpz$%!BWrmzzSA&igkB2`qxi4ue z2d(By{8Nlc{CCVJmHwo&@z=a&=}g(#KtS~cXgqRRsF*snBjrGxzrfsZ zf84c-y)`wGa-oSLLMVV=vkiBNj;%&nF>mC ztklQAj2y=Gu3mFc3vbH5C>{lGFUN*~zS(I#Xlq@!fq51zR#|Ztvqj1VvbVT=er(Cm zCg~*8@|1$ix(H?MtB1m}>C=U%BOV?C2ia3cA?mgF$Jf>_vLyCYx6z`8*vmB0-0P~C zsuLsSk^nbuMplMEeD7PYN`(a9?1jmSFHqj=W+@t%0!r}Jlr37LwsVF#MFh-s&R}wGC-wN~P zbx1ukAx_-fP?B&z(m2jjFf1F|^rN{z+0(6yq?HNp=-GJosfZwDv{p+c5vvs;naze) z9Bes;OaRk0+In+SIR@0Ei51p?V4Pr`U!)ZB{!zOq{$42r(=mRLZ^s?umE76)`wHm} zu4y+=%CsHhX>3d4#$A}yx}&84!TX1`^R2X}@XZFnEIZm*Wb83=PS~yJDGv=0@>P8p z2bQNJ!RIzcuo@`%?FC!x$b4}jgW=8&#s<6VixukLv2!RJb?lm#D_?*|klU{&HyAtD zx!UAQAH`3A5z&gw~QJv17Sc>bf$6<}~1X>*o*2>B6<*`4f2EhLI z#h!~y#z6^Fd#sTYuUjpxso`vi5>I(z-WlRF?P3zj_=!#~e(^*3xB`0IM(yb|U~q^WJYCd>MYvm9e! zgLJL8%<|)^9<{KnMSmfDnypewf>0lgnGQiu$;x`!;F!ODLn;7RnSoe~C&;jPtx%N< z_)(A33r44$Lh)?n5T~tsh0ZogV}qc5vXW`aPFE`((--im9Co%o!fdz&L&kF3Zxpn> z#Zuz4{{VVWtD*d)q2+>Y^6iuE<&v643S>`fQkcof*|7Gj)T_(~)Mu|#s|ah!<`TPB z>1^4Zt3wKd+vn?Ik_xCmC48aAr(5?g^To!+0IuVLO*SgcpAMfz=}Tyfn~=`#-(8|w zzI-lhSLr#hx2;dCX@L!b68X)9s)bdmKXc_orTY;Xp2!6hkk4<<2d7c3^3FaP4FQGm z!*sJkrx_Snl}JxOp_AU?;-@Cf!A}bnhvMyP#FSG>gh(5DQf5^Y?T6fDQfM3-p1mm7 z+S0^QpE5bjrkYPv%1n__n9zHtIQVyW8Tw&{ZWP6Co4b_|O~cjC6@<^$e%M}(*woCw zkT$ka_{z|ar`ETgw6>4$!183xStoivONIz;^2f-GJ8sjk2d?js(wt4g2Nf%oRbR9O1hR-sdMCc#aRKL9Y!{i zjF$6X8lrGe0nd44SLfz%Y#q!MBU+V zy9_c*%8jE?Fk_-wGk;o9R4=bC&`x$;~^A0l|w?_QFkDppn z0S?32QQffa+&)aQ5@U(;@2G6ndBnI(y>lYY;5es=2HWoaF`N=z78QINo!rIuL$%4D zBEs(G(r)o2(q&Nd|?!0>}ZcIk10%+q8dN=5FVT`HS)>u#% zn_BkDhx9YU!3=As_p+qDcyD><0DWj3$uvRhbo!Ey>ucQjs#ZRm4@UL^s$~!8^ zE`GfM4*DxQ7rNJ>#m`0c^NN^S`3`DzvBQ2(B`N74)hdBKB8|^d;twJC9{QFsa_tH` zlP-Qso9V%aoD;<)G+iXK5YwHZH~aN`2QzLC*+he;HGcb6KOPDLs`)c;fkBanPSKGW zPy)45A&VE|W6Z8#DWJt0HQPj{paAaa-?oTene8x>D_QD6w^bP%NE zxz>B@-t|8H{~yje>-_fK=i`PVh-t8Z*iZz~JPsaroNC+^1o#M8h9E+M5JZE2a32%_ zZvB4}_$>$lF8zZG|4c3p!T+TQ@qt)n2(auQC<%4=Cjo{)&7flco`g^y5Jm7aWzBb1 zrG&&n-~o8{(~B={pn7 zSYnbTSpN_V=>#D_vlU}S(&+N9w^u%GUWu+5!-jN^2YZ;EfQovg-=WCJ1Ad$N9x8Mm znaR7XRdwMeAJi7+*x_oi+dCQQ{*hziSedsPzw*7Js0gs^v5b1suR=-D)KdOU6I@Q#U9{+`VE9Y8zS6{BI{ zV`oIXcE#0-lVsY%9(Nu)2~|2&7&SWK{UjQ4du=AM+<&C?lYulCAE)?Frl{K6S4QrE zB_XPv-xuP@;=S#1VOYD#ROL)0#K6cV-wB1jS)?}Fv&-0M^!2D~o|k12`|o@dyD`2c zU$B69_5s1m&x=c!bWBQmdnb}-mDbW++O%w=n&lg2?dy1sx=XMy~32qzd|4B5AfhE%@=^Oqe0Wd-$9=n(7j64Up9<>>4aYO z2rV-8cA$7QL#L`h4)aXQu%rHWt&yK@#^a#2jwLT)N)u74Cn01#~U-hS+sYj$JEhs64gn+pU`htaeT$zJo4_B&U$rh>dd#= zEuKk=#wz?Q1ehc9T8~nxNlOXmFc=4Capd1zQdIHhOpkoEVxS+vlM|8G-jc|uB~HzM zJ-sQ^pXOsf-Klx_Iz!f_uimOTSzc2Etyf;hnm*Y`rSr5Q0a&B@gI&Rap{T+wJcIyx zWL0oqh?ACRyLGG1xV3U2+*E85xl)|;aOX~~bn;3hBGqV9`EJxSY_!jWAYdmhvs}`D z!E|Q!rafjtc41zO<2u|U1^p_QNs>p(kNEMul;U?FP&H_=IJ+v3cgv>|J;?qo470RP z5Kf!+tHi&7Qa|_Ad+ZmrF3lc9&E-=1&hJybpKiC>F8HJ$-8P zjt;RifW|J=%?;sr!m%It^GmzPXdu*upYvV~+$W$2 zW%{Qs@hAky7)byg08fA)zy{zC@CCpD_E4684UYvVKxBr_?r>%S7z`>1g^CHo#6^Xn z2t43Frbjqs2;3h~1nvnI5b$U1{;L50Q&kI7@~09YrE=y?m{#&-T|s9415wgIM2xKW&6(96T@eAd7$Knpo--y=R(( zQP4vg;u2>3PU7a?k}n1$kj|3VYVENYe%{)ZXMhY{j zQq5q&85iSWAt0Pdrg*jdLR@p$p7Q8&>+3XX3(uGLqXUxunfg1|k}fQ1t#-{hB5-}^ zXz!4}4N~_7a)d^DYmk<&PyNyfTcU(nRvFejZLs_HLd%U;K2UA-L0=TVqW+z$+Tn2C@lcA%^B!9=Xi)Xw@qTEHePau!YS)$3FTNS3l9f0SZUB=lNmTA7w zO7>aR{rvvyhrAy{PvgtZmz?PeY zBNHq~O)t;)J(Ht1?Rj5(n6lj`XiftxpYrTfk+xk2WH zY{egTgxsp1R_aaNXbt3qM~t2*I#}5iH_UK;gnCKL)ghMI!_h$>0@gF(t zNK)}rA=WYcA_wXGdH|4I`V611tdPW9e&Ou6n=&7SBv zwj9adyBUL2A$g6$ZZ7*Jr+iA%<3fpVuuBUJH-9HM(%Qo-_RBJIam&9~*{LA1dYh(6 zxatY0`7f8p)PDtq47t|A-_ETq;1jgu^qD_3QpUvXuyImL(4)E}c8!gm427n&ahA=j z1^IO5oePO@3jNGsdse~lS!^*eVHxWY?l#ApxZ!@}|Th68s; z)Aq*BRf>tMZ^Qk#h3M>v)HF$JP6m;#V=!P50y(%p9xVv;2@=@-;CP2BYQwzD%3ZeF z*N^SobTZt|@YbtS&~Co#j=t`?2&d9x5F`|rUukHp|0`9o9D5*`0l7*X_hW440L}+edtLExfn2K@x_4bmUq*HJ3MD@F9x$6*nY(EB8r8H}Hw`Lk|55v_&$tMU zITFSylSw4=;Y0aJtPSORWGbfN@AC__q*s?NCak))q>2_Ol_6HB3{bz5FNsOKesSh};ETXBcvR#*!MSaj$eC=opLbnx!-J=+5o8WsKB? zlFX7j-;Oc6tgOYwvz?p!sx z-|-PnC-s}J)bW#&*vh_Y5?~y_Vp#(mL)sE>OhuJm7LYxxdjuuTr?QN>9}39KYm;ER zNo_NmmD1w|BN8Ejd@L%A<|-~PDDAk7Mo0=3D>+RSb{*YB>&UmIvct_r4PunsAc0I@ zCc9D!h``ygQ<6EuB1|#unSe%7{iRJS)aJ)kiKi37TXbJnIQ%3x7MgsfoF5=y#=P|t zcxt2plxAShwinlqKUOFygF{v8FD5JPjK-OCfR&YP7j}_P0rYvDVYk#Np=+H!&YX2WHE025g3$nUiYU_{bj<5}*O6k|u@)`#1Pi z{=V$8a=15h#FwsaH=OXUu5UAm#njjo0z^DJ6soE8q(kQFLDTpOd@5!E*ZFqqo&{YG zQt$ANuZPxKD@b07mSNx@wqztuJtOCTL1dGX=Ej3!a5ab$tA%CP1haEE(96?6zfpyv zQt5JSv*j{69Dxq2W7N4A5GCTombR}IbJ(lhWg-~cn7S6wr8VC}gko~mfVpj;4ZR6n z<^v0Jz`-!83O7y|7Rka&&OzOab)2d;F2BEdUVDDY{TLOXW5^ekFX{1vruUUUQ*WJi zc3Y`ycZ%zw<2L7pKZONdZFat;IfleYnxVRE!Bix(iCWe_jA9e=uEVFHzgvOw{0 zz&u3QShQdOh{8=@CJOQ?F|+mMe#tiB>}QyG+zN=IjsH_VwGGM(P9IFgjG!rh&7L%i Jg$;}=`yXAMCD8x? literal 0 HcmV?d00001 diff --git a/datafilecollector/config/truststore.pass b/datafilecollector/config/truststore.pass new file mode 100755 index 0000000..b915b0f --- /dev/null +++ b/datafilecollector/config/truststore.pass @@ -0,0 +1 @@ +policy_agent \ No newline at end of file diff --git a/datafilecollector/onap-java-formatter.xml b/datafilecollector/onap-java-formatter.xml new file mode 100644 index 0000000..ccd07d9 --- /dev/null +++ b/datafilecollector/onap-java-formatter.xml @@ -0,0 +1,296 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/datafilecollector/pom.xml b/datafilecollector/pom.xml new file mode 100644 index 0000000..c122253 --- /dev/null +++ b/datafilecollector/pom.xml @@ -0,0 +1,334 @@ + + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.0.4 + + org.o-ran-sc.nonrtric.plt.ranpm + datafile-app-server + jar + + 17 + yyyyMMdd'T'HHmmss + 8.7.1 + 3.0.0 + 2.9.0 + 0.30.0 + 0.8.8 + + + + com.google.code.gson + gson + 2.9.1 + + + io.projectreactor.kafka + reactor-kafka + 1.3.9 + + + org.projectlombok + lombok + + + org.awaitility + awaitility + test + + + org.apache.httpcomponents + httpasyncclient + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.boot + spring-boot + + + org.springframework + spring-web + + + org.springframework + spring-webmvc + + + com.spotify + docker-client + ${docker-client.version} + + + org.springframework + spring-webflux + + + org.springframework.boot + spring-boot-autoconfigure + + + org.apache.httpcomponents.core5 + httpcore5 + + + + org.springframework.boot + spring-boot-starter-actuator + + + software.amazon.awssdk + s3 + 2.17.292 + + + + io.projectreactor + reactor-test + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + 5.7.0 + test + + + org.mockito + mockito-junit-jupiter + test + + + + io.springfox + springfox-spring-web + ${springfox.version} + + + io.springfox + springfox-spi + ${springfox.version} + + + io.springfox + springfox-core + ${springfox.version} + + + org.springdoc + springdoc-openapi-ui + 1.6.3 + test + + + io.springfox + springfox-swagger2 + ${springfox.version} + + + io.springfox + springfox-swagger-ui + ${springfox.version} + + + commons-net + commons-net + 3.6 + + + com.jcraft + jsch + 0.1.55 + + + io.projectreactor.netty + reactor-netty + 1.0.22 + + + + + + org.springframework.boot + spring-boot-maven-plugin + + ${project.artifactId} + org.onap.dcaegen2.collectors.datafile.MainApp + + + + + build-info + + + + + + io.fabric8 + docker-maven-plugin + ${docker-maven-plugin} + false + + + generate-image + package + + build + + + ${env.CONTAINER_PULL_REGISTRY} + + + + o-ran-sc/nonrtric-plt-ranpm-datafilecollector:${project.version} + + try + ${basedir} + Dockerfile + + ${project.build.finalName}.jar + + + ${project.version} + + + + + + + + push-image + + build + push + + + ${env.CONTAINER_PULL_REGISTRY} + ${env.CONTAINER_PUSH_REGISTRY} + + + + o-ran-sc/nonrtric-plt-ranpm-datafilecollector:${project.version} + + ${basedir} + Dockerfile + + ${project.build.finalName}.jar + + + ${project.version} + latest + + + + + + + + + + pl.project13.maven + git-commit-id-plugin + + + get-the-git-infos + + revision + + + + + true + ${project.basedir}/.git + MM-dd-yyyy '@' HH:mm:ss Z + true + ${project.build.outputDirectory}/git.properties + + true + + + + com.diffplug.spotless + spotless-maven-plugin + 1.18.0 + + + + + com,java,javax,org + + + + + + + net.revelc.code.formatter + formatter-maven-plugin + 2.8.1 + + ${project.basedir}/onap-java-formatter.xml + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + + default-prepare-agent + + prepare-agent + + + + default-report + prepare-package + + report + + + + + + + \ No newline at end of file diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/MainApp.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/MainApp.java new file mode 100644 index 0000000..851db32 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/MainApp.java @@ -0,0 +1,47 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 NOKIA Intellectual Property. All rights reserved. + * Copyright (C) 2018-2021 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.concurrent.ConcurrentTaskScheduler; + +/** + * The main app of DFC. + * + * @author Przemysław Wąsala on 3/23/18 + * @author Henrik Andersson + */ +@SpringBootApplication() +@EnableScheduling +public class MainApp { + + public static void main(String[] args) { + SpringApplication.run(MainApp.class, args); + } + + @Bean + TaskScheduler taskScheduler() { + return new ConcurrentTaskScheduler(); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileCollectClient.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileCollectClient.java new file mode 100644 index 0000000..517e382 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileCollectClient.java @@ -0,0 +1,32 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.commons; + +import java.nio.file.Path; + +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; + +/** + * A closeable file client. + * + * @author Henrik Andersson + */ +public interface FileCollectClient extends AutoCloseable { + public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException; + + public void open() throws DatafileTaskException; +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java new file mode 100644 index 0000000..5896fe6 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/FileServerData.java @@ -0,0 +1,51 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Modifications copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.commons; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Builder; +import lombok.ToString; + +import org.apache.hc.core5.http.NameValuePair; + +/** + * Data about the file server to collect a file from. + * In case of http protocol it also contains data required to recreate target + * uri + */ +@Builder +@ToString +public class FileServerData { + + public String serverAddress; + public String userId; + + @ToString.Exclude + public String password; + + @Builder.Default + @ToString.Exclude + public List queryParameters = new ArrayList<>(); + + @Builder.Default + public String uriRawFragment = ""; + + public Integer port; +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java new file mode 100644 index 0000000..613fa39 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/Scheme.java @@ -0,0 +1,70 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.commons; + +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; + +/** + * Enum specifying the schemes that DFC support for downloading files. + * + * @author Henrik Andersson + * + */ +public enum Scheme { + FTPES, SFTP, HTTP, HTTPS; + + public static final String DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG = "DFC does not support protocol "; + public static final String SUPPORTED_PROTOCOLS_ERROR_MESSAGE = + ". Supported protocols are FTPeS, sFTP, HTTP and HTTPS"; + + /** + * Get a Scheme from a string. + * + * @param schemeString the string to convert to Scheme. + * @return The corresponding Scheme + * @throws DatafileTaskException if the value of the string doesn't match any defined scheme. + */ + public static Scheme getSchemeFromString(String schemeString) throws DatafileTaskException { + Scheme result; + if ("FTPES".equalsIgnoreCase(schemeString)) { + result = Scheme.FTPES; + } else if ("SFTP".equalsIgnoreCase(schemeString)) { + result = Scheme.SFTP; + } else if ("HTTP".equalsIgnoreCase(schemeString)) { + result = Scheme.HTTP; + } else if ("HTTPS".equalsIgnoreCase(schemeString)) { + result = Scheme.HTTPS; + } else { + throw new DatafileTaskException( + DFC_DOES_NOT_SUPPORT_PROTOCOL_ERROR_MSG + schemeString + SUPPORTED_PROTOCOLS_ERROR_MESSAGE); + } + return result; + } + + /** + * Check if Scheme is FTP type or HTTP type. + * + * @param scheme the Scheme which has to be checked. + * @return true if Scheme is FTP type or false if it is HTTP type + */ + public static boolean isFtpScheme(Scheme scheme) { + return scheme == SFTP || scheme == FTPES; + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java new file mode 100644 index 0000000..9d6b7f9 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/commons/SecurityUtil.java @@ -0,0 +1,52 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.commons; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class containing functions used for certificates configuration + * + * @author Krzysztof Gajewski + */ +public final class SecurityUtil { + private SecurityUtil() { + } + + private static final Logger logger = LoggerFactory.getLogger(SecurityUtil.class); + + public static String getKeystorePasswordFromFile(String passwordPath) { + return getPasswordFromFile(passwordPath, "Keystore"); + } + + public static String getTruststorePasswordFromFile(String passwordPath) { + return getPasswordFromFile(passwordPath, "Truststore"); + } + + public static String getPasswordFromFile(String passwordPath, String element) { + try { + return new String(Files.readAllBytes(Paths.get(passwordPath))); + } catch (IOException e) { + logger.error("{} password file at path: {} cannot be opened ", element, passwordPath); + } + return ""; + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java new file mode 100644 index 0000000..f8be04d --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/AppConfig.java @@ -0,0 +1,126 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018, 2020-2022 Nokia. All rights reserved. + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.configuration; + +import java.util.Properties; + +import lombok.Getter; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Holds all configuration for the DFC. + * + * @author Przemysław Wąsala on + * 3/23/18 + * @author Henrik Andersson + */ + +@Component +@EnableConfigurationProperties +public class AppConfig { + + @Value("#{systemEnvironment}") + Properties systemEnvironment; + + @Value("${app.filepath}") + String filepath; + + @Value("${app.kafka.bootstrap-servers:}") + private String kafkaBootStrapServers; + + @Value("${app.kafka.collected-file-topic:}") + public String collectedFileTopic; + + @Value("${app.kafka.file-ready-event-topic:}") + public String fileReadyEventTopic; + + @Value("${app.kafka.client-id:undefined}") + public String kafkaClientId; + + @Value("${app.collected-files-path:}") + public String collectedFilesPath; + + @Value("${app.sftp.strict-host-key-checking:false}") + public boolean strictHostKeyChecking; + + @Value("${app.sftp.known-hosts-file-path:}") + public String knownHostsFilePath; + + @Value("${app.ssl.key-store-password-file}") + private String clientKeyStorePassword = ""; + + @Value("${app.ssl.key-store:}") + private String clientKeyStore = ""; + + @Value("${app.ssl.trust-store:}") + private String clientTrustStore = ""; + + @Value("${app.ssl.trust-store-password-file:}") + private String clientTrustStorePassword; + + @Getter + @Value("${app.s3.endpointOverride:}") + private String s3EndpointOverride; + + @Getter + @Value("${app.s3.accessKeyId:}") + private String s3AccessKeyId; + + @Getter + @Value("${app.s3.secretAccessKey:}") + private String s3SecretAccessKey; + + @Getter + @Value("${app.s3.bucket:}") + private String s3Bucket; + + @Value("${app.s3.locksBucket:}") + private String s3LocksBucket; + + public String getS3LocksBucket() { + return s3LocksBucket.isEmpty() ? s3Bucket : s3LocksBucket; + } + + public boolean isS3Enabled() { + return !s3EndpointOverride.isEmpty() && !s3Bucket.isEmpty(); + } + + public String getKafkaBootStrapServers() { + return kafkaBootStrapServers; + } + + public synchronized CertificateConfig getCertificateConfiguration() { + return CertificateConfig.builder() // + .trustedCa(this.clientTrustStore) // + .trustedCaPasswordPath(this.clientTrustStorePassword) // + .keyCert(this.clientKeyStore) // + .keyPasswordPath(this.clientKeyStorePassword) // + .build(); + } + + public synchronized SftpConfig getSftpConfiguration() { + return SftpConfig.builder() // + .knownHostsFilePath(this.knownHostsFilePath) // + .strictHostKeyChecking(this.strictHostKeyChecking) // + .build(); + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java new file mode 100644 index 0000000..938d322 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/CertificateConfig.java @@ -0,0 +1,36 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2018-2022 Nokia. All rights reserved. + * Copyright (C) 2019 Nordix Foundation. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.configuration; + +import lombok.Builder; + +@Builder +public class CertificateConfig { + + public String keyCert; + + public String keyPasswordPath; + + public String trustedCa; + + public String trustedCaPasswordPath; +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SftpConfig.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SftpConfig.java new file mode 100644 index 0000000..182a59e --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SftpConfig.java @@ -0,0 +1,31 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2020 NOKIA Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.configuration; + +import lombok.Builder; + +@Builder +public class SftpConfig { + + public boolean strictHostKeyChecking; + + public String knownHostsFilePath; +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SwaggerConfig.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SwaggerConfig.java new file mode 100644 index 0000000..b7dc521 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/configuration/SwaggerConfig.java @@ -0,0 +1,42 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 NOKIA Intellectual Property. All rights reserved. + * Copyright (C) 2018-2021 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.configuration; + +import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.info.Info; +import io.swagger.v3.oas.annotations.info.License; + +@OpenAPIDefinition( + info = @Info( + title = SwaggerConfig.API_TITLE, + version = SwaggerConfig.VERSION, + description = SwaggerConfig.DESCRIPTION, + license = @License( + name = "Copyright (C) 2020 Nordix Foundation. Licensed under the Apache License.", + url = "http://www.apache.org/licenses/LICENSE-2.0"))) +public class SwaggerConfig { + + public static final String VERSION = "1.0"; + public static final String API_TITLE = "DATAFILE App Server"; + static final String DESCRIPTION = "

This page lists all the rest apis for DATAFILE app server.

"; + + private SwaggerConfig() { + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusController.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusController.java new file mode 100644 index 0000000..d1f615b --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusController.java @@ -0,0 +1,99 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 NOKIA Intellectual Property. All rights reserved. + * Copyright (C) 2018-2021 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.controllers; + +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +import org.onap.dcaegen2.collectors.datafile.model.Counters; +import org.onap.dcaegen2.collectors.datafile.tasks.CollectAndReportFiles; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Mono; + +/** + * REST Controller to check the heart beat and status of the DFC. + */ +@RestController +@Tag(name = "StatusController") +public class StatusController { + + private static final Logger logger = LoggerFactory.getLogger(StatusController.class); + + private final CollectAndReportFiles collectAndReportFiles; + + public StatusController(CollectAndReportFiles task) { + this.collectAndReportFiles = task; + } + + /** + * Checks the heart beat of DFC. + * + * @return the heart beat status of DFC. + */ + @GetMapping("/heartbeat") + @Operation(summary = "Returns liveness of DATAFILE service") + @ApiResponses( + value = { // + @ApiResponse(code = 200, message = "DATAFILE service is living"), + @ApiResponse(code = 401, message = "You are not authorized to view the resource"), + @ApiResponse(code = 403, message = "Accessing the resource you were trying to reach is forbidden"), + @ApiResponse(code = 404, message = "The resource you were trying to reach is not found")}) + public Mono> heartbeat(@RequestHeader HttpHeaders headers) { + logger.info("ENTRY {}", "Heartbeat request"); + + String statusString = "I'm living!"; + + Mono> response = Mono.just(new ResponseEntity<>(statusString, HttpStatus.OK)); + logger.info("EXIT {}", "Heartbeat request"); + return response; + } + + /** + * Returns diagnostics and statistics information. It is intended for testing + * and trouble + * shooting. + * + * @return information. + */ + @GetMapping("/status") + @Operation(summary = "Returns status and statistics of DATAFILE service") + @ApiResponses( + value = { // + @ApiResponse(code = 200, message = "DATAFILE service is living"), + @ApiResponse(code = 401, message = "You are not authorized to view the resource"), + @ApiResponse(code = 403, message = "Accessing the resource you were trying to reach is forbidden"), + @ApiResponse(code = 404, message = "The resource you were trying to reach is not found")}) + public Mono> status(@RequestHeader HttpHeaders headers) { + + logger.info("ENTRY {}", "Status request"); + + Counters counters = collectAndReportFiles.getCounters(); + Mono> response = Mono.just(new ResponseEntity<>(counters.toString(), HttpStatus.OK)); + logger.info("EXIT {}", "Status request"); + return response; + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/DataStore.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/DataStore.java new file mode 100644 index 0000000..af0512e --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/DataStore.java @@ -0,0 +1,57 @@ +/*- + * ========================LICENSE_START================================= + * O-RAN-SC + * %% + * Copyright (C) 2021 Nordix Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================LICENSE_END=================================== + */ + +package org.onap.dcaegen2.collectors.datafile.datastore; + +import java.nio.file.Path; + +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public interface DataStore { + public enum Bucket { + FILES, LOCKS + } + + public Flux listObjects(Bucket bucket, String prefix); + + public Mono readObject(Bucket bucket, String name); + + public Mono createLock(String name); + + public Mono deleteLock(String name); + + public Mono deleteObject(Bucket bucket, String name); + + public Mono copyFileTo(Path from, String to); + + public Mono create(DataStore.Bucket bucket); + + public Mono deleteBucket(Bucket bucket); + + public Mono fileExists(Bucket bucket, String key); + + public static DataStore create(AppConfig config) { + return config.isS3Enabled() ? new S3ObjectStore(config) : new FileStore(config); + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/FileStore.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/FileStore.java new file mode 100644 index 0000000..7f497be --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/FileStore.java @@ -0,0 +1,160 @@ +/*- + * ========================LICENSE_START================================= + * O-RAN-SC + * %% + * Copyright (C) 2021 Nordix Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================LICENSE_END=================================== + */ + +package org.onap.dcaegen2.collectors.datafile.datastore; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.FileSystemUtils; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class FileStore implements DataStore { + private static final Logger logger = LoggerFactory.getLogger(java.lang.invoke.MethodHandles.lookup().lookupClass()); + + AppConfig applicationConfig; + + public FileStore(AppConfig applicationConfig) { + this.applicationConfig = applicationConfig; + } + + @Override + public Flux listObjects(Bucket bucket, String prefix) { + Path root = Path.of(applicationConfig.collectedFilesPath, prefix); + if (!root.toFile().exists()) { + root = root.getParent(); + } + + logger.debug("Listing files in: {}", root); + + List result = new ArrayList<>(); + try (Stream stream = Files.walk(root, Integer.MAX_VALUE)) { + + stream.forEach(path -> filterListFiles(path, prefix, result)); + + return Flux.fromIterable(result); + } catch (Exception e) { + return Flux.error(e); + } + } + + private void filterListFiles(Path path, String prefix, List result) { + if (path.toFile().isFile() && externalName(path).startsWith(prefix)) { + result.add(externalName(path)); + } else { + logger.debug("Ignoring file {} that does not start with: {}", path, prefix); + } + } + + private String externalName(Path path) { + String fullName = path.toString(); + String externalName = fullName.substring(applicationConfig.collectedFilesPath.length()); + if (externalName.startsWith("/")) { + externalName = externalName.substring(1); + } + return externalName; + } + + @Override + public Mono readObject(Bucket bucket, String fileName) { + try { + byte[] contents = Files.readAllBytes(path(fileName)); + return Mono.just(contents); + } catch (Exception e) { + return Mono.error(e); + } + } + + @Override + public Mono createLock(String name) { + File file = path(name).toFile(); + try { + Files.createDirectories(path(name).getParent()); + boolean res = file.createNewFile(); + return Mono.just(res); + } catch (Exception e) { + logger.warn("Could not create lock file: {}, reason: {}", file.getPath(), e.getMessage()); + return Mono.just(!file.exists()); + } + } + + @Override + public Mono copyFileTo(Path from, String to) { + try { + Path toPath = path(to); + Files.createDirectories(toPath); + Files.copy(from, path(to), StandardCopyOption.REPLACE_EXISTING); + return Mono.just(to); + } catch (Exception e) { + return Mono.error(e); + } + } + + @Override + public Mono deleteLock(String name) { + return deleteObject(Bucket.LOCKS, name); + } + + @Override + public Mono deleteObject(Bucket bucket, String name) { + try { + Files.delete(path(name)); + return Mono.just(true); + } catch (Exception e) { + return Mono.just(false); + } + } + + @Override + public Mono create(Bucket bucket) { + return Mono.just("OK"); + } + + private Path path(String name) { + return Path.of(applicationConfig.collectedFilesPath, name); + } + + public Mono fileExists(Bucket bucket, String key) { + return Mono.just(path(key).toFile().exists()); + } + + @Override + public Mono deleteBucket(Bucket bucket) { + try { + FileSystemUtils.deleteRecursively(Path.of(applicationConfig.collectedFilesPath)); + } catch (IOException e) { + logger.debug("Could not delete directory: {}, reason; {}", applicationConfig.collectedFilesPath, + e.getMessage()); + } + return Mono.just("OK"); + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/S3ObjectStore.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/S3ObjectStore.java new file mode 100644 index 0000000..f93bbaf --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/datastore/S3ObjectStore.java @@ -0,0 +1,313 @@ +/*- + * ========================LICENSE_START================================= + * O-RAN-SC + * %% + * Copyright (C) 2021 Nordix Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================LICENSE_END=================================== + */ + +package org.onap.dcaegen2.collectors.datafile.datastore; + +import java.net.URI; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; + +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.BytesWrapper; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.CreateBucketResponse; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteBucketResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Object; + +public class S3ObjectStore implements DataStore { + private static final Logger logger = LoggerFactory.getLogger(S3ObjectStore.class); + private final AppConfig applicationConfig; + + private static S3AsyncClient s3AsynchClient; + + public S3ObjectStore(AppConfig applicationConfig) { + this.applicationConfig = applicationConfig; + + getS3AsynchClient(applicationConfig); + } + + private static synchronized S3AsyncClient getS3AsynchClient(AppConfig applicationConfig) { + if (applicationConfig.isS3Enabled() && s3AsynchClient == null) { + s3AsynchClient = getS3AsyncClientBuilder(applicationConfig).build(); + } + return s3AsynchClient; + } + + private static S3AsyncClientBuilder getS3AsyncClientBuilder(AppConfig applicationConfig) { + URI uri = URI.create(applicationConfig.getS3EndpointOverride()); + return S3AsyncClient.builder() // + .region(Region.US_EAST_1) // + .endpointOverride(uri) // + .credentialsProvider(StaticCredentialsProvider.create( // + AwsBasicCredentials.create(applicationConfig.getS3AccessKeyId(), // + applicationConfig.getS3SecretAccessKey()))); + } + + @Override + public Flux listObjects(Bucket bucket, String prefix) { + return listObjectsInBucket(bucket(bucket), prefix).map(S3Object::key); + } + + @Override + public Mono createLock(String name) { + return getHeadObject(bucket(Bucket.LOCKS), name).flatMap(head -> createLock(name, head)) // + .onErrorResume(t -> createLock(name, null)); + } + + private Mono createLock(String name, HeadObjectResponse head) { + if (head == null) { + + return this.putObject(Bucket.LOCKS, name, "") // + .flatMap(resp -> Mono.just(true)) // + .doOnError(t -> logger.warn("Failed to create lock {}, reason: {}", name, t.getMessage())) // + .onErrorResume(t -> Mono.just(false)); + } else { + return Mono.just(false); + } + } + + @Override + public Mono deleteLock(String name) { + return deleteObject(Bucket.LOCKS, name); + } + + @Override + public Mono deleteObject(Bucket bucket, String name) { + + DeleteObjectRequest request = DeleteObjectRequest.builder() // + .bucket(bucket(bucket)) // + .key(name) // + .build(); + + CompletableFuture future = s3AsynchClient.deleteObject(request); + + return Mono.fromFuture(future).map(resp -> true); + } + + @Override + public Mono readObject(Bucket bucket, String fileName) { + return getDataFromS3Object(bucket(bucket), fileName); + } + + public Mono putObject(Bucket bucket, String fileName, String bodyString) { + PutObjectRequest request = PutObjectRequest.builder() // + .bucket(bucket(bucket)) // + .key(fileName) // + .build(); + + AsyncRequestBody body = AsyncRequestBody.fromString(bodyString); + + CompletableFuture future = s3AsynchClient.putObject(request, body); + + return Mono.fromFuture(future) // + .map(putObjectResponse -> fileName) // + .doOnError(t -> logger.error("Failed to store file in S3 {}", t.getMessage())); + } + + @Override + public Mono copyFileTo(Path fromFile, String toFile) { + return copyFileToS3Bucket(bucket(Bucket.FILES), fromFile, toFile); + } + + public Mono fileExists(Bucket bucket, String key) { + return this.getHeadObject(bucket(bucket), key).map(obj -> true) // + .onErrorResume(t -> Mono.just(false)); + } + + @Override + public Mono create(Bucket bucket) { + return createS3Bucket(bucket(bucket)); + } + + private Mono createS3Bucket(String s3Bucket) { + + CreateBucketRequest request = CreateBucketRequest.builder() // + .bucket(s3Bucket) // + .build(); + + CompletableFuture future = s3AsynchClient.createBucket(request); + + return Mono.fromFuture(future) // + .map(f -> s3Bucket) // + .doOnError(t -> logger.trace("Could not create S3 bucket: {}", t.getMessage())) + .onErrorResume(t -> Mono.just(s3Bucket)); + } + + @Override + public Mono deleteBucket(Bucket bucket) { + return deleteAllFiles(bucket) // + .collectList() // + .flatMap(list -> deleteBucketFromS3Storage(bucket)) // + .map(resp -> "OK") + .doOnError(t -> logger.warn("Could not delete bucket: {}, reason: {}", bucket(bucket), t.getMessage())) + .onErrorResume(t -> Mono.just("NOK")); + } + + private Flux deleteAllFiles(Bucket bucket) { + return listObjectsInBucket(bucket(bucket), "") // + .buffer(500) // + .flatMap(list -> deleteObjectsFromS3Storage(bucket, list)) // + .doOnError(t -> logger.info("Deleted all files {}", t.getMessage())) // + .onErrorStop() // + .onErrorResume(t -> Flux.empty()); // + + } + + private Mono deleteObjectsFromS3Storage(Bucket bucket, Collection objects) { + Collection oids = new ArrayList<>(); + for (S3Object o : objects) { + ObjectIdentifier oid = ObjectIdentifier.builder() // + .key(o.key()) // + .build(); + oids.add(oid); + } + + Delete delete = Delete.builder() // + .objects(oids) // + .build(); + + DeleteObjectsRequest request = DeleteObjectsRequest.builder() // + .bucket(bucket(bucket)) // + .delete(delete) // + .build(); + + CompletableFuture future = s3AsynchClient.deleteObjects(request); + + return Mono.fromFuture(future); + } + + private Mono listObjectsRequest(String bucket, String prefix, + ListObjectsResponse prevResponse) { + ListObjectsRequest.Builder builder = ListObjectsRequest.builder() // + .bucket(bucket) // + .maxKeys(1000) // + .prefix(prefix); + + if (prevResponse != null) { + if (Boolean.TRUE.equals(prevResponse.isTruncated())) { + builder.marker(prevResponse.nextMarker()); + } else { + return Mono.empty(); + } + } + + ListObjectsRequest listObjectsRequest = builder.build(); + CompletableFuture future = s3AsynchClient.listObjects(listObjectsRequest); + return Mono.fromFuture(future); + } + + private Flux listObjectsInBucket(String bucket, String prefix) { + + return listObjectsRequest(bucket, prefix, null) // + .expand(response -> listObjectsRequest(bucket, prefix, response)) // + .map(ListObjectsResponse::contents) // + .doOnNext(f -> logger.debug("Found objects in {}: {}", bucket, f.size())) // + .doOnError(t -> logger.warn("Error fromlist objects: {}", t.getMessage())) // + .flatMap(Flux::fromIterable) // + .doOnNext(obj -> logger.debug("Found object: {}", obj.key())); + } + + private Mono deleteBucketFromS3Storage(Bucket bucket) { + DeleteBucketRequest request = DeleteBucketRequest.builder() // + .bucket(bucket(bucket)) // + .build(); + + CompletableFuture future = s3AsynchClient.deleteBucket(request); + + return Mono.fromFuture(future); + } + + private String bucket(Bucket bucket) { + return bucket == Bucket.FILES ? applicationConfig.getS3Bucket() : applicationConfig.getS3LocksBucket(); + } + + private Mono copyFileToS3Bucket(String s3Bucket, Path fileName, String s3Key) { + + PutObjectRequest request = PutObjectRequest.builder() // + .bucket(s3Bucket) // + .key(s3Key) // + .build(); + + AsyncRequestBody body = AsyncRequestBody.fromFile(fileName); + + CompletableFuture future = s3AsynchClient.putObject(request, body); + + return Mono.fromFuture(future) // + .map(f -> s3Key) // + .doOnError(t -> logger.error("Failed to store file in S3 {}", t.getMessage())); + + } + + private Mono getHeadObject(String bucket, String key) { + HeadObjectRequest request = HeadObjectRequest.builder().bucket(bucket).key(key).build(); + + CompletableFuture future = s3AsynchClient.headObject(request); + return Mono.fromFuture(future); + } + + private Mono getDataFromS3Object(String bucket, String key) { + + GetObjectRequest request = GetObjectRequest.builder() // + .bucket(bucket) // + .key(key) // + .build(); + + CompletableFuture> future = + s3AsynchClient.getObject(request, AsyncResponseTransformer.toBytes()); + + return Mono.fromFuture(future) // + .map(BytesWrapper::asByteArray) // + .doOnError( + t -> logger.error("Failed to get file from S3, key:{}, bucket: {}, {}", key, bucket, t.getMessage())) // + .doOnNext(n -> logger.debug("Read file from S3: {} {}", bucket, key)) // + .onErrorResume(t -> Mono.empty()); + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/DatafileTaskException.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/DatafileTaskException.java new file mode 100644 index 0000000..6aa7615 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/DatafileTaskException.java @@ -0,0 +1,33 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.exceptions; + +public class DatafileTaskException extends Exception { + + private static final long serialVersionUID = 1L; + + public DatafileTaskException(String message) { + super(message); + } + + public DatafileTaskException(String message, Exception originalException) { + super(message, originalException); + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/EnvironmentLoaderException.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/EnvironmentLoaderException.java new file mode 100644 index 0000000..d49a051 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/EnvironmentLoaderException.java @@ -0,0 +1,33 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2018 NOKIA Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.exceptions; + +/** + * Exception thrown when there is a problem with the Consul environment. + * + * @author Przemysław Wąsala on 9/19/18 + */ +public class EnvironmentLoaderException extends Exception { + + private static final long serialVersionUID = 1L; + + public EnvironmentLoaderException(String message) { + super(message); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/NonRetryableDatafileTaskException.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/NonRetryableDatafileTaskException.java new file mode 100644 index 0000000..5c2a0d2 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/exceptions/NonRetryableDatafileTaskException.java @@ -0,0 +1,33 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2019 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.exceptions; + +public class NonRetryableDatafileTaskException extends DatafileTaskException { + + private static final long serialVersionUID = 1L; + + public NonRetryableDatafileTaskException(String message) { + super(message); + } + + public NonRetryableDatafileTaskException(String message, Exception originalException) { + super(message, originalException); + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java new file mode 100644 index 0000000..aef5033 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClient.java @@ -0,0 +1,230 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2020-2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; + +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPReply; +import org.apache.commons.net.ftp.FTPSClient; +import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.FileSystemResource; + +/** + * Gets file from PNF with FTPS protocol. + * + * @author Martin Yan + */ +public class FtpesClient implements FileCollectClient { + private static final Logger logger = LoggerFactory.getLogger(FtpesClient.class); + + private static final int DEFAULT_PORT = 21; + + FTPSClient realFtpsClient = new FTPSClient(); + private final FileServerData fileServerData; + private static TrustManager theTrustManager = null; + private static KeyManager theKeyManager = null; + + private final Path keyCertPath; + private final String keyCertPasswordPath; + private final Path trustedCaPath; + private final String trustedCaPasswordPath; + + /** + * Constructor. + * + * @param fileServerData info needed to connect to the PNF. + * @param keyCertPath path to DFC's key cert. + * @param keyCertPasswordPath path of file containing password for DFC's key + * cert. + * @param trustedCaPath path to the PNF's trusted keystore. + * @param trustedCaPasswordPath path of file containing password for the PNF's + * trusted keystore. + */ + public FtpesClient(FileServerData fileServerData, Path keyCertPath, String keyCertPasswordPath, Path trustedCaPath, + String trustedCaPasswordPath) { + this.fileServerData = fileServerData; + this.keyCertPath = keyCertPath; + this.keyCertPasswordPath = keyCertPasswordPath; + this.trustedCaPath = trustedCaPath; + this.trustedCaPasswordPath = trustedCaPasswordPath; + } + + @Override + public void open() throws DatafileTaskException { + try { + realFtpsClient.setNeedClientAuth(trustedCaPath != null); + realFtpsClient.setKeyManager(getKeyManager(keyCertPath, keyCertPasswordPath)); + realFtpsClient.setTrustManager(getTrustManager(trustedCaPath, trustedCaPasswordPath)); + setUpConnection(); + } catch (DatafileTaskException e) { + throw e; + } catch (Exception e) { + throw new DatafileTaskException("Could not open connection: " + e, e); + } + } + + @Override + public void close() { + logger.trace("starting to closeDownConnection"); + if (realFtpsClient.isConnected()) { + try { + boolean logOut = realFtpsClient.logout(); + logger.trace("logOut: {}", logOut); + } catch (Exception e) { + logger.trace("Unable to logout connection.", e); + } + try { + realFtpsClient.disconnect(); + logger.trace("disconnected!"); + } catch (Exception e) { + logger.trace("Unable to disconnect connection.", e); + } + } + } + + @Override + public void collectFile(String remoteFileName, Path localFileName) throws DatafileTaskException { + logger.trace("collectFile called"); + + try (OutputStream output = createOutputStream(localFileName)) { + logger.trace("begin to retrieve from xNF."); + if (!realFtpsClient.retrieveFile(remoteFileName, output)) { + throw new NonRetryableDatafileTaskException( + "Could not retrieve file. No retry attempts will be done, file :" + remoteFileName); + } + } catch (IOException e) { + throw new DatafileTaskException("Could not fetch file: " + e, e); + } + logger.trace("collectFile fetched: {}", localFileName); + } + + private static int getPort(Integer port) { + return port != null ? port : DEFAULT_PORT; + } + + private void setUpConnection() throws DatafileTaskException, IOException { + + realFtpsClient.connect(fileServerData.serverAddress, getPort(fileServerData.port)); + logger.trace("after ftp connect"); + + if (!realFtpsClient.login(fileServerData.userId, fileServerData.password)) { + throw new DatafileTaskException("Unable to log in to xNF. " + fileServerData.serverAddress); + } + + if (FTPReply.isPositiveCompletion(realFtpsClient.getReplyCode())) { + realFtpsClient.enterLocalPassiveMode(); + realFtpsClient.setFileType(FTP.BINARY_FILE_TYPE); + // Set protection buffer size + realFtpsClient.execPBSZ(0); + // Set data channel protection to private + realFtpsClient.execPROT("P"); + realFtpsClient.setBufferSize(1024 * 1024); + } else { + throw new DatafileTaskException("Unable to connect to xNF. " + fileServerData.serverAddress + + " xNF reply code: " + realFtpsClient.getReplyCode()); + } + + logger.trace("setUpConnection successfully!"); + } + + private TrustManager createTrustManager(Path trustedCaPath, String trustedCaPassword) + throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException { + logger.trace("Creating trust manager from file: {}", trustedCaPath); + try (InputStream fis = createInputStream(trustedCaPath)) { + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(fis, trustedCaPassword.toCharArray()); + TrustManagerFactory factory = TrustManagerFactory.getInstance("SunX509"); + factory.init(keyStore); + return factory.getTrustManagers()[0]; + } + } + + protected InputStream createInputStream(Path localFileName) throws IOException { + FileSystemResource realResource = new FileSystemResource(localFileName); + return realResource.getInputStream(); + } + + protected OutputStream createOutputStream(Path localFileName) throws IOException, DatafileTaskException { + File localFile = localFileName.toFile(); + if (!localFile.createNewFile()) { + logger.debug("Local file {} already created", localFileName); + throw new NonRetryableDatafileTaskException("Local file already created: " + localFileName); + } + OutputStream output = new FileOutputStream(localFile); + logger.trace("File {} opened xNF", localFileName); + return output; + } + + protected TrustManager getTrustManager(Path trustedCaPath, String trustedCaPasswordPath) + throws KeyStoreException, NoSuchAlgorithmException, IOException, CertificateException { + synchronized (FtpesClient.class) { + if (theTrustManager == null && trustedCaPath != null) { + String trustedCaPassword = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath); + theTrustManager = createTrustManager(trustedCaPath, trustedCaPassword); + } + return theTrustManager; + } + } + + protected KeyManager getKeyManager(Path keyCertPath, String keyCertPasswordPath) + throws IOException, GeneralSecurityException { + + synchronized (FtpesClient.class) { + if (theKeyManager == null) { + String keyCertPassword = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath); + theKeyManager = createKeyManager(keyCertPath, keyCertPassword); + } + return theKeyManager; + } + } + + private KeyManager createKeyManager(Path keyCertPath, String keyCertPassword) throws IOException, KeyStoreException, + NoSuchAlgorithmException, CertificateException, UnrecoverableKeyException { + logger.trace("Creating key manager from file: {}", keyCertPath); + try (InputStream fis = createInputStream(keyCertPath)) { + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(fis, keyCertPassword.toCharArray()); + KeyManagerFactory factory = KeyManagerFactory.getInstance("SunX509"); + factory.init(keyStore, keyCertPassword.toCharArray()); + return factory.getKeyManagers()[0]; + } + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java new file mode 100644 index 0000000..0c6db35 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClient.java @@ -0,0 +1,151 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation, 2020 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import com.jcraft.jsch.Channel; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpException; + +import java.nio.file.Path; + +import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Gets file from xNF with SFTP protocol. + * + * @author Martin Yan + */ +public class SftpClient implements FileCollectClient { + + private static final Logger logger = LoggerFactory.getLogger(SftpClient.class); + + private static final int SFTP_DEFAULT_PORT = 22; + private static final String STRICT_HOST_KEY_CHECKING = "StrictHostKeyChecking"; + + private final FileServerData fileServerData; + protected Session session = null; + protected ChannelSftp sftpChannel = null; + private final SftpClientSettings settings; + + public SftpClient(FileServerData fileServerData, SftpClientSettings sftpConfig) { + this.fileServerData = fileServerData; + this.settings = sftpConfig; + } + + @Override + public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { + logger.trace("collectFile {}", localFile); + + try { + sftpChannel.get(remoteFile, localFile.toString()); + logger.trace("File {} Download successful from xNF", localFile.getFileName()); + } catch (SftpException e) { + boolean retry = e.id != ChannelSftp.SSH_FX_NO_SUCH_FILE && e.id != ChannelSftp.SSH_FX_PERMISSION_DENIED + && e.id != ChannelSftp.SSH_FX_OP_UNSUPPORTED; + if (retry) { + throw new DatafileTaskException("Unable to get file from xNF. Data: " + fileServerData, e); + } else { + throw new NonRetryableDatafileTaskException( + "Unable to get file from xNF. No retry attempts will be done. Data: " + fileServerData, e); + } + } + + logger.trace("collectFile OK"); + } + + @Override + public void close() { + logger.trace("closing sftp session"); + if (sftpChannel != null) { + sftpChannel.exit(); + sftpChannel = null; + } + if (session != null) { + session.disconnect(); + session = null; + } + } + + @Override + public void open() throws DatafileTaskException { + try { + if (session == null) { + session = setUpSession(fileServerData); + sftpChannel = getChannel(session); + } + } catch (JSchException e) { + boolean retry = !e.getMessage().contains("Auth fail"); + if (retry) { + throw new DatafileTaskException("Could not open Sftp client. " + e); + } else { + throw new NonRetryableDatafileTaskException( + "Could not open Sftp client, no retry attempts will be done. " + e); + } + } + } + + JSch createJsch() { + return new JSch(); + } + + private int getPort(Integer port) { + return port != null ? port : SFTP_DEFAULT_PORT; + } + + private Session setUpSession(FileServerData fileServerData) throws JSchException { + boolean useStrictHostChecking = this.settings.shouldUseStrictHostChecking(); + JSch jsch = createJschClient(useStrictHostChecking); + return createJshSession(jsch, fileServerData, useStrictHostChecking); + } + + private JSch createJschClient(boolean useStrictHostChecking) throws JSchException { + JSch jsch = createJsch(); + if (useStrictHostChecking) { + jsch.setKnownHosts(this.settings.getKnownHostsFilePath()); + } + return jsch; + } + + private Session createJshSession(JSch jsch, FileServerData fileServerData, boolean useStrictHostKeyChecking) + throws JSchException { + Session newSession = + jsch.getSession(fileServerData.userId, fileServerData.serverAddress, getPort(fileServerData.port)); + newSession.setConfig(STRICT_HOST_KEY_CHECKING, toYesNo(useStrictHostKeyChecking)); + newSession.setPassword(fileServerData.password); + newSession.connect(); + return newSession; + } + + private String toYesNo(boolean useStrictHostKeyChecking) { + return useStrictHostKeyChecking ? "yes" : "no"; + } + + private ChannelSftp getChannel(Session session) throws JSchException { + Channel channel = session.openChannel("sftp"); + channel.connect(); + return (ChannelSftp) channel; + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettings.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettings.java new file mode 100644 index 0000000..23e254b --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettings.java @@ -0,0 +1,60 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2020 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import java.io.File; + +import org.onap.dcaegen2.collectors.datafile.configuration.SftpConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SftpClientSettings { + + private static final Logger logger = LoggerFactory.getLogger(SftpClientSettings.class); + + private final SftpConfig sftpConfig; + + public SftpClientSettings(SftpConfig sftpConfig) { + this.sftpConfig = sftpConfig; + } + + public boolean shouldUseStrictHostChecking() { + boolean strictHostKeyChecking = false; + if (this.sftpConfig.strictHostKeyChecking) { + File file = new File(getKnownHostsFilePath()); + strictHostKeyChecking = file.isFile(); + logUsageOfStrictHostCheckingFlag(strictHostKeyChecking, file.getAbsolutePath()); + } else { + logger.info("StrictHostKeyChecking will be disabled."); + } + return strictHostKeyChecking; + } + + public String getKnownHostsFilePath() { + return this.sftpConfig.knownHostsFilePath; + } + + private void logUsageOfStrictHostCheckingFlag(boolean strictHostKeyChecking, String filePath) { + if (strictHostKeyChecking) { + logger.info("StrictHostKeyChecking will be enabled with KNOWN_HOSTS_FILE_PATH [{}].", filePath); + } else { + logger.warn( + "StrictHostKeyChecking is enabled but environment variable KNOWN_HOSTS_FILE_PATH is not set or points to not existing file [{}] --> falling back to StrictHostKeyChecking='no'.", + filePath); + } + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java new file mode 100644 index 0000000..eab0082 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClient.java @@ -0,0 +1,179 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2020-2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.service.HttpUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; +import reactor.netty.http.client.HttpClientResponse; +import reactor.netty.resources.ConnectionProvider; + +/** + * Gets file from PNF with HTTP protocol. + * + * @author Krzysztof Gajewski + */ +public class DfcHttpClient implements FileCollectClient { + + // Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS + private static final int MAX_NUMBER_OF_CONNECTIONS = 200; + private static final Logger logger = LoggerFactory.getLogger(DfcHttpClient.class); + private static final ConnectionProvider pool = ConnectionProvider.create("default", MAX_NUMBER_OF_CONNECTIONS); + + private final FileServerData fileServerData; + private Disposable disposableClient; + + protected HttpClient client; + + public DfcHttpClient(FileServerData fileServerData) { + this.fileServerData = fileServerData; + } + + @Override + public void open() throws DatafileTaskException { + logger.trace("Setting httpClient for file download."); + + String authorizationContent = getAuthorizationContent(); + this.client = + HttpClient.create(pool).keepAlive(true).headers(h -> h.add("Authorization", authorizationContent)); + + logger.trace("httpClient, auth header was set."); + } + + protected String getAuthorizationContent() throws DatafileTaskException { + String jwtToken = HttpUtils.getJWTToken(fileServerData); + if (!jwtToken.isEmpty()) { + return HttpUtils.jwtAuthContent(jwtToken); + } + if (!HttpUtils.isBasicAuthDataFilled(fileServerData)) { + throw new DatafileTaskException("Not sufficient basic auth data for file."); + } + return HttpUtils.basicAuthContent(this.fileServerData.userId, this.fileServerData.password); + } + + @Override + public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { + logger.trace("Prepare to collectFile {}", localFile); + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference errorMessage = new AtomicReference<>(); + + Consumer onError = processFailedConnectionWithServer(latch, errorMessage); + Consumer onSuccess = processDataFromServer(localFile, latch, errorMessage); + + Flux responseContent = getServerResponse(remoteFile); + disposableClient = responseContent.subscribe(onSuccess, onError); + + try { + latch.await(); + } catch (InterruptedException e) { + throw new DatafileTaskException("Interrupted exception after datafile download - ", e); + } + + if (isDownloadFailed(errorMessage)) { + if (errorMessage.get() instanceof NonRetryableDatafileTaskException) { + throw (NonRetryableDatafileTaskException) errorMessage.get(); + } + throw (DatafileTaskException) errorMessage.get(); + } + + logger.trace("HTTP collectFile OK"); + } + + protected boolean isDownloadFailed(AtomicReference errorMessage) { + return (errorMessage.get() != null); + } + + protected Consumer processFailedConnectionWithServer(CountDownLatch latch, + AtomicReference errorMessages) { + return (Throwable response) -> { + Exception e = new Exception("Error in connection has occurred during file download", response); + errorMessages.set(new DatafileTaskException(response.getMessage(), e)); + if (response instanceof NonRetryableDatafileTaskException) { + errorMessages.set(new NonRetryableDatafileTaskException(response.getMessage(), e)); + } + latch.countDown(); + }; + } + + protected Consumer processDataFromServer(Path localFile, CountDownLatch latch, + AtomicReference errorMessages) { + return (InputStream response) -> { + logger.trace("Starting to process response."); + try { + long numBytes = Files.copy(response, localFile); + logger.trace("Transmission was successful - {} bytes downloaded.", numBytes); + logger.trace("CollectFile fetched: {}", localFile); + response.close(); + } catch (IOException e) { + errorMessages.set(new DatafileTaskException("Error fetching file with", e)); + } finally { + latch.countDown(); + } + }; + } + + protected Flux getServerResponse(String remoteFile) { + return client.get().uri(HttpUtils.prepareHttpUri(fileServerData, remoteFile)) + .response((responseReceiver, byteBufFlux) -> { + logger.trace("HTTP response status - {}", responseReceiver.status()); + if (isResponseOk(responseReceiver)) { + return byteBufFlux.aggregate().asInputStream(); + } + if (isErrorInConnection(responseReceiver)) { + return Mono.error(new NonRetryableDatafileTaskException( + HttpUtils.nonRetryableResponse(getResponseCode(responseReceiver)))); + } + return Mono + .error(new DatafileTaskException(HttpUtils.retryableResponse(getResponseCode(responseReceiver)))); + }); + } + + protected boolean isResponseOk(HttpClientResponse httpClientResponse) { + return getResponseCode(httpClientResponse) == 200; + } + + private int getResponseCode(HttpClientResponse responseReceiver) { + return responseReceiver.status().code(); + } + + protected boolean isErrorInConnection(HttpClientResponse httpClientResponse) { + return getResponseCode(httpClientResponse) >= 400; + } + + @Override + public void close() { + logger.trace("Starting http client disposal."); + disposableClient.dispose(); + logger.trace("Http client disposed."); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java new file mode 100644 index 0000000..5c652cb --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClient.java @@ -0,0 +1,182 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import java.io.IOException; +import java.io.InputStream; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.util.EntityUtils; +import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.service.HttpUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Gets file from PNF with HTTPS protocol. + * + * @author Krzysztof Gajewski + */ +public class DfcHttpsClient implements FileCollectClient { + + protected CloseableHttpClient httpsClient; + + private static final Logger logger = LoggerFactory.getLogger(DfcHttpsClient.class); + private static final int FIFTEEN_SECONDS = 15 * 1000; + + private final FileServerData fileServerData; + private final PoolingHttpClientConnectionManager connectionManager; + + public DfcHttpsClient(FileServerData fileServerData, PoolingHttpClientConnectionManager connectionManager) { + this.fileServerData = fileServerData; + this.connectionManager = connectionManager; + } + + @Override + public void open() { + logger.trace("Setting httpsClient for file download."); + SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).build(); + + RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(FIFTEEN_SECONDS).build(); + + httpsClient = HttpClients.custom().setConnectionManager(connectionManager).setDefaultSocketConfig(socketConfig) + .setDefaultRequestConfig(requestConfig).build(); + + logger.trace("httpsClient prepared for connection."); + } + + @Override + public void collectFile(String remoteFile, Path localFile) throws DatafileTaskException { + logger.trace("Prepare to collectFile {}", localFile); + HttpGet httpGet = new HttpGet(HttpUtils.prepareHttpsUri(fileServerData, remoteFile)); + + String authorizationContent = getAuthorizationContent(); + if (!authorizationContent.isEmpty()) { + httpGet.addHeader("Authorization", authorizationContent); + } + try { + HttpResponse httpResponse = makeCall(httpGet); + processResponse(httpResponse, localFile); + } catch (IOException e) { + logger.error("marker", e); + throw new DatafileTaskException("Error downloading file from server. ", e); + } + logger.trace("HTTPS collectFile OK"); + } + + private String getAuthorizationContent() throws DatafileTaskException { + String jwtToken = HttpUtils.getJWTToken(fileServerData); + if (shouldUseBasicAuth(jwtToken)) { + return HttpUtils.basicAuthContent(this.fileServerData.userId, this.fileServerData.password); + } + return HttpUtils.jwtAuthContent(jwtToken); + } + + private boolean shouldUseBasicAuth(String jwtToken) throws DatafileTaskException { + return basicAuthValidNotPresentOrThrow() && jwtToken.isEmpty(); + } + + protected boolean basicAuthValidNotPresentOrThrow() throws DatafileTaskException { + if (isAuthDataEmpty()) { + return false; + } + if (HttpUtils.isBasicAuthDataFilled(fileServerData)) { + return true; + } + throw new DatafileTaskException("Not sufficient basic auth data for file."); + } + + private boolean isAuthDataEmpty() { + return this.fileServerData.userId.isEmpty() && this.fileServerData.password.isEmpty(); + } + + protected HttpResponse makeCall(HttpGet httpGet) throws IOException, DatafileTaskException { + try { + HttpResponse httpResponse = executeHttpClient(httpGet); + if (isResponseOk(httpResponse)) { + return httpResponse; + } + + EntityUtils.consume(httpResponse.getEntity()); + if (isErrorInConnection(httpResponse)) { + logger.warn("Failed to download file, reason: {}, code: {}", + httpResponse.getStatusLine().getReasonPhrase(), httpResponse.getStatusLine()); + throw new NonRetryableDatafileTaskException(HttpUtils.retryableResponse(getResponseCode(httpResponse))); + } + throw new DatafileTaskException(HttpUtils.nonRetryableResponse(getResponseCode(httpResponse))); + } catch (ConnectTimeoutException | UnknownHostException | HttpHostConnectException | SSLHandshakeException + | SSLPeerUnverifiedException e) { + logger.warn("Unable to get file from xNF: {}", e.getMessage()); + throw new NonRetryableDatafileTaskException("Unable to get file from xNF. No retry attempts will be done.", + e); + } + } + + protected CloseableHttpResponse executeHttpClient(HttpGet httpGet) throws IOException { + return httpsClient.execute(httpGet); + } + + protected boolean isResponseOk(HttpResponse httpResponse) { + return getResponseCode(httpResponse) == 200; + } + + private int getResponseCode(HttpResponse httpResponse) { + return httpResponse.getStatusLine().getStatusCode(); + } + + protected boolean isErrorInConnection(HttpResponse httpResponse) { + return getResponseCode(httpResponse) >= 400; + } + + protected void processResponse(HttpResponse response, Path localFile) throws IOException { + logger.trace("Starting to process response."); + HttpEntity entity = response.getEntity(); + InputStream stream = entity.getContent(); + long numBytes = writeFile(localFile, stream); + stream.close(); + EntityUtils.consume(entity); + logger.trace("Transmission was successful - {} bytes downloaded.", numBytes); + } + + protected long writeFile(Path localFile, InputStream stream) throws IOException { + return Files.copy(stream, localFile, StandardCopyOption.REPLACE_EXISTING); + } + + @Override + public void close() { + logger.trace("Https client has ended downloading process."); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpAsyncClientBuilderWrapper.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpAsyncClientBuilderWrapper.java new file mode 100644 index 0000000..b003727 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpAsyncClientBuilderWrapper.java @@ -0,0 +1,59 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.http; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; + +import org.apache.http.client.RedirectStrategy; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.impl.nio.client.HttpAsyncClients; + +public class HttpAsyncClientBuilderWrapper { + HttpAsyncClientBuilder builder = HttpAsyncClients.custom(); + + public HttpAsyncClientBuilderWrapper setRedirectStrategy(RedirectStrategy redirectStrategy) { + builder.setRedirectStrategy(redirectStrategy); + return this; + } + + public HttpAsyncClientBuilderWrapper setSslContext(SSLContext sslcontext) { + builder.setSSLContext(sslcontext); + return this; + } + + public HttpAsyncClientBuilderWrapper setSslHostnameVerifier(HostnameVerifier hostnameVerifier) { + builder.setSSLHostnameVerifier(hostnameVerifier); + return this; + } + + public HttpAsyncClientBuilderWrapper setDefaultRequestConfig(RequestConfig config) { + builder.setDefaultRequestConfig(config); + return this; + } + + public CloseableHttpAsyncClient build() { + return builder.build(); + } + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java new file mode 100644 index 0000000..7769e53 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtil.java @@ -0,0 +1,138 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; + +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; +import org.onap.dcaegen2.collectors.datafile.commons.SecurityUtil; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.FileSystemResource; + +/** + * Utility class supplying connection manager for HTTPS protocol. + * + * @author Krzysztof Gajewski + */ +public class HttpsClientConnectionManagerUtil { + + private HttpsClientConnectionManagerUtil() { + } + + private static final Logger logger = LoggerFactory.getLogger(HttpsClientConnectionManagerUtil.class); + // Be aware to be less than ScheduledTasks.NUMBER_OF_WORKER_THREADS + private static final int MAX_NUMBER_OF_CONNECTIONS = 200; + private static PoolingHttpClientConnectionManager connectionManager; + + public static PoolingHttpClientConnectionManager instance() throws DatafileTaskException { + if (connectionManager == null) { + throw new DatafileTaskException("ConnectionManager has to be set or update first"); + } + return connectionManager; + } + + public static void setupOrUpdate(String keyCertPath, String keyCertPasswordPath, String trustedCaPath, + String trustedCaPasswordPath, boolean useHostnameVerifier) throws DatafileTaskException { + synchronized (HttpsClientConnectionManagerUtil.class) { + if (connectionManager != null) { + connectionManager.close(); + connectionManager = null; + } + setup(keyCertPath, keyCertPasswordPath, trustedCaPath, trustedCaPasswordPath, useHostnameVerifier); + } + logger.trace("HttpsConnectionManager setup or updated"); + } + + private static void setup(String keyCertPath, String keyCertPasswordPath, String trustedCaPath, + String trustedCaPasswordPath, boolean useHostnameVerifier) throws DatafileTaskException { + try { + SSLContextBuilder sslBuilder = SSLContexts.custom(); + sslBuilder = supplyKeyInfo(keyCertPath, keyCertPasswordPath, sslBuilder); + if (!trustedCaPath.isEmpty()) { + sslBuilder = supplyTrustInfo(trustedCaPath, trustedCaPasswordPath, sslBuilder); + } + + SSLContext sslContext = sslBuilder.build(); + + HostnameVerifier hostnameVerifier = + useHostnameVerifier ? new DefaultHostnameVerifier() : NoopHostnameVerifier.INSTANCE; + + SSLConnectionSocketFactory sslConnectionSocketFactory = + new SSLConnectionSocketFactory(sslContext, new String[] {"TLSv1.2"}, null, hostnameVerifier); + + Registry socketFactoryRegistry = + RegistryBuilder.create().register("https", sslConnectionSocketFactory).build(); + + connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connectionManager.setMaxTotal(MAX_NUMBER_OF_CONNECTIONS); + + } catch (Exception e) { + throw new DatafileTaskException("Unable to prepare HttpsConnectionManager : ", e); + } + } + + private static SSLContextBuilder supplyKeyInfo(String keyCertPath, String keyCertPasswordPath, + SSLContextBuilder sslBuilder) throws IOException, KeyStoreException, NoSuchAlgorithmException, + CertificateException, UnrecoverableKeyException { + String keyPass = SecurityUtil.getKeystorePasswordFromFile(keyCertPasswordPath); + KeyStore keyFile = createKeyStore(keyCertPath, keyPass); + return sslBuilder.loadKeyMaterial(keyFile, keyPass.toCharArray()); + } + + private static KeyStore createKeyStore(String path, String storePassword) + throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException { + logger.trace("Creating manager from file: {}", path); + try (InputStream fis = createInputStream(path)) { + KeyStore keyStore = KeyStore.getInstance("PKCS12"); + keyStore.load(fis, storePassword.toCharArray()); + return keyStore; + } + } + + private static InputStream createInputStream(String localFileName) throws IOException { + FileSystemResource realResource = new FileSystemResource(Paths.get(localFileName)); + return realResource.getInputStream(); + } + + private static SSLContextBuilder supplyTrustInfo(String trustedCaPath, String trustedCaPasswordPath, + SSLContextBuilder sslBuilder) + throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException { + String trustPass = SecurityUtil.getTruststorePasswordFromFile(trustedCaPasswordPath); + File trustStoreFile = new File(trustedCaPath); + return sslBuilder.loadTrustMaterial(trustStoreFile, trustPass.toCharArray()); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java new file mode 100644 index 0000000..2323d0e --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/Counters.java @@ -0,0 +1,145 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.model; + +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Various counters that can be shown via a REST API. + * + */ +public class Counters { + + private long noOfCollectedFiles = 0; + private long noOfFailedFtpAttempts = 0; + private long noOfFailedHttpAttempts = 0; + private long noOfFailedFtp = 0; + private long noOfFailedHttp = 0; + private long noOfFailedPublishAttempts = 0; + private long totalPublishedFiles = 0; + private long noOfFailedPublish = 0; + private Instant lastPublishedTime = Instant.MIN; + private long totalReceivedEvents = 0; + private Instant lastEventTime = Instant.MIN; + + public final AtomicInteger threadPoolQueueSize = new AtomicInteger(); + + public synchronized void incNoOfReceivedEvents() { + totalReceivedEvents++; + lastEventTime = Instant.now(); + } + + public synchronized void incNoOfCollectedFiles() { + noOfCollectedFiles++; + } + + public synchronized void incNoOfFailedFtpAttempts() { + noOfFailedFtpAttempts++; + } + + public synchronized void incNoOfFailedHttpAttempts() { + noOfFailedHttpAttempts++; + } + + public synchronized void incNoOfFailedFtp() { + noOfFailedFtp++; + } + + public synchronized void incNoOfFailedHttp() { + noOfFailedHttp++; + } + + public synchronized void incNoOfFailedPublishAttempts() { + noOfFailedPublishAttempts++; + } + + public synchronized void incTotalPublishedFiles() { + totalPublishedFiles++; + lastPublishedTime = Instant.now(); + } + + public synchronized void incNoOfFailedPublish() { + noOfFailedPublish++; + } + + @Override + public synchronized String toString() { + StringBuilder str = new StringBuilder(); + str.append(format("totalReceivedEvents", totalReceivedEvents)); + str.append(format("lastEventTime", lastEventTime)); + str.append("\n"); + str.append(format("collectedFiles", noOfCollectedFiles)); + str.append(format("failedFtpAttempts", noOfFailedFtpAttempts)); + str.append(format("failedHttpAttempts", noOfFailedHttpAttempts)); + str.append(format("failedFtp", noOfFailedFtp)); + str.append(format("failedHttp", noOfFailedHttp)); + str.append("\n"); + str.append(format("totalPublishedFiles", totalPublishedFiles)); + str.append(format("lastPublishedTime", lastPublishedTime)); + + str.append(format("failedPublishAttempts", noOfFailedPublishAttempts)); + str.append(format("noOfFailedPublish", noOfFailedPublish)); + + return str.toString(); + } + + private static String format(String name, Object value) { + String header = name + ":"; + return String.format("%-24s%-22s%n", header, value); + } + + public long getNoOfCollectedFiles() { + return noOfCollectedFiles; + } + + public long getNoOfFailedFtpAttempts() { + return noOfFailedFtpAttempts; + } + + public long getNoOfFailedHttpAttempts() { + return noOfFailedHttpAttempts; + } + + public long getNoOfFailedFtp() { + return noOfFailedFtp; + } + + public long getNoOfFailedHttp() { + return noOfFailedHttp; + } + + public long getNoOfFailedPublishAttempts() { + return noOfFailedPublishAttempts; + } + + public long getTotalPublishedFiles() { + return totalPublishedFiles; + } + + public long getNoOfFailedPublish() { + return noOfFailedPublish; + } + + public long getTotalReceivedEvents() { + return totalReceivedEvents; + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java new file mode 100644 index 0000000..3de5817 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileData.java @@ -0,0 +1,162 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. + * Copyright (C) 2021 Nokia. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.model; + +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import lombok.Builder; + +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.net.URIBuilder; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData.FileServerDataBuilder; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Contains data, from the fileReady event, about the file to collect from the + * xNF. + * + */ +@Builder +public class FileData { + + private static final Logger logger = LoggerFactory.getLogger(FileData.class); + + public FileReadyMessage.ArrayOfNamedHashMap fileInfo; + + public FileReadyMessage.MessageMetaData messageMetaData; + + public static Iterable createFileData(FileReadyMessage msg) { + Collection res = new ArrayList<>(); + for (FileReadyMessage.ArrayOfNamedHashMap arr : msg.event.notificationFields.arrayOfNamedHashMap) { + FileData data = FileData.builder().fileInfo(arr).messageMetaData(msg.event.commonEventHeader).build(); + res.add(data); + } + return res; + } + + /** + * Get the name of the PNF, must be unique in the network. + * + * @return the name of the PNF, must be unique in the network + */ + public String sourceName() { + return messageMetaData.sourceName; + } + + public String name() { + return this.messageMetaData.sourceName + "/" + fileInfo.name; + } + + /** + * Get the path to file to get from the PNF. + * + * @return the path to the file on the PNF. + */ + public String remoteFilePath() { + return URI.create(fileInfo.hashMap.location).getPath(); + } + + public Scheme scheme() { + URI uri = URI.create(fileInfo.hashMap.location); + try { + return Scheme.getSchemeFromString(uri.getScheme()); + } catch (Exception e) { + logger.warn("Could noit get scheme :{}", e.getMessage()); + return Scheme.FTPES; + } + } + + /** + * Get the path to the locally stored file. + * + * @return the path to the locally stored file. + */ + public Path getLocalFilePath(AppConfig config) { + return Paths.get(config.collectedFilesPath, this.messageMetaData.sourceName, fileInfo.name); + } + + /** + * Get the data about the file server where the file should be collected from. + * Query data included as it can contain JWT token + * + * @return the data about the file server where the file should be collected + * from. + */ + public FileServerData fileServerData() { + URI uri = URI.create(fileInfo.hashMap.location); + Optional userInfo = getUserNameAndPasswordIfGiven(uri.getUserInfo()); + + FileServerDataBuilder builder = FileServerData.builder() // + .serverAddress(uri.getHost()) // + .userId(userInfo.isPresent() ? userInfo.get()[0] : "") // + .password(userInfo.isPresent() ? userInfo.get()[1] : ""); + if (uri.getPort() > 0) { + builder.port(uri.getPort()); + } + URIBuilder uriBuilder = new URIBuilder(uri); + List query = uriBuilder.getQueryParams(); + if (query != null && !query.isEmpty()) { + builder.queryParameters(query); + } + String fragment = uri.getRawFragment(); + if (fragment != null && fragment.length() > 0) { + builder.uriRawFragment(fragment); + } + return builder.build(); + } + + /** + * Extracts user name and password from the user info, if it they are given in + * the URI. + * + * @param userInfoString the user info string from the URI. + * + * @return An Optional containing a String array with the user name + * and password if given, or an empty + * Optional if not given. + */ + private static Optional getUserNameAndPasswordIfGiven(String userInfoString) { + if (userInfoString != null) { + String[] userAndPassword = userInfoString.split(":"); + if (userAndPassword.length == 2) { + return Optional.of(userAndPassword); + } else if (userAndPassword.length == 1)// if just user + { + String[] tab = new String[2]; + tab[0] = userAndPassword[0]; + tab[1] = "";// add empty password + return Optional.of(tab); + } + } + return Optional.empty(); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FilePublishInformation.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FilePublishInformation.java new file mode 100644 index 0000000..52e6413 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FilePublishInformation.java @@ -0,0 +1,52 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.model; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +@Builder +@EqualsAndHashCode +public class FilePublishInformation { + + String productName; + + String vendorName; + + long lastEpochMicrosec; + + @Getter + String sourceName; + + long startEpochMicrosec; + + String timeZoneOffset; + + String compression; + + String fileFormatType; + + String fileFormatVersion; + + @Getter + String name; + + String changeIdentifier; + + String objectStoreBucket; +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileReadyMessage.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileReadyMessage.java new file mode 100644 index 0000000..9ee461e --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/model/FileReadyMessage.java @@ -0,0 +1,111 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.model; + +import java.util.List; + +import lombok.Builder; + +@Builder +public class FileReadyMessage { + + /** + * Meta data about a fileReady message. + */ + @Builder + public static class MessageMetaData { + + public String eventId; + + public String priority; + public String version; + public String reportingEntityName; + public int sequence; + public String domain; + + public String eventName; + public String vesEventListenerVersion; + + public String sourceName; + + public long lastEpochMicrosec; + public long startEpochMicrosec; + + public String timeZoneOffset; + + public String changeIdentifier; + + /** + * Gets data from the event name. Defined as: + * {DomainAbbreviation}_{productName}-{vendorName}_{Description}, + * example: Noti_RnNode-Ericsson_FileReady + * + */ + public String productName() { + String[] eventArray = eventName.split("_|-"); + if (eventArray.length >= 2) { + return eventArray[1]; + } else { + return eventName; + } + } + + public String vendorName() { + String[] eventArray = eventName.split("_|-"); + if (eventArray.length >= 3) { + return eventArray[2]; + } else { + return eventName; + } + } + } + + @Builder + public static class FileInfo { + public String fileFormatType; + public String location; + public String fileFormatVersion; + public String compression; + } + + @Builder + public static class ArrayOfNamedHashMap { + public String name; + public FileInfo hashMap; + } + + @Builder + public static class NotificationFields { + public String notificationFieldsVersion; + public String changeType; + public String changeIdentifier; + public List arrayOfNamedHashMap; + } + + @Builder + public static class Event { + public MessageMetaData commonEventHeader; + public NotificationFields notificationFields; + } + + public Event event; + +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java new file mode 100644 index 0000000..208691f --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtils.java @@ -0,0 +1,229 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Modifications Copyright (C) 2020-2021 Nokia. All rights reserved + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.service; + +import java.util.Base64; +import java.util.List; + +import org.apache.hc.core5.http.NameValuePair; +import org.apache.http.HttpStatus; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class HttpUtils implements HttpStatus { + + private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); + public static final int HTTP_DEFAULT_PORT = 80; + public static final int HTTPS_DEFAULT_PORT = 443; + public static final String JWT_TOKEN_NAME = "access_token"; + public static final String AUTH_JWT_WARN = "Both JWT token and Basic auth data present. Omitting basic auth info."; + public static final String AUTH_JWT_ERROR = + "More than one JWT token present in the queryParameters. Omitting JWT token."; + + private HttpUtils() { + } + + public static String nonRetryableResponse(int responseCode) { + return "Unexpected response code - " + responseCode; + } + + public static String retryableResponse(int responseCode) { + return "Unexpected response code - " + responseCode + ". No retry attempts will be done."; + } + + public static boolean isSuccessfulResponseCodeWithDataRouter(Integer statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + public static boolean isBasicAuthDataFilled(final FileServerData fileServerData) { + return !fileServerData.userId.isEmpty() && !fileServerData.password.isEmpty(); + } + + public static String basicAuthContent(String username, String password) { + return "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()); + } + + public static String jwtAuthContent(String token) { + return "Bearer " + token; + } + + /** + * Prepare uri to retrieve file from xNF using HTTP connection. If JWT token was + * included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param fileServerData fileServerData including - server address, port, + * queryParameters and uriRawFragment + * @param remoteFile file which has to be downloaded + * @return uri String representing the xNF HTTP location + */ + public static String prepareHttpUri(FileServerData fileServerData, String remoteFile) { + return prepareUri("http", fileServerData, remoteFile, HTTP_DEFAULT_PORT); + } + + /** + * Prepare uri to retrieve file from xNF using HTTPS connection. If JWT token + * was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param fileServerData fileServerData including - server address, port, + * queryParameters and uriRawFragment + * @param remoteFile file which has to be downloaded + * @return uri String representing the xNF HTTPS location + */ + public static String prepareHttpsUri(FileServerData fileServerData, String remoteFile) { + return prepareUri("https", fileServerData, remoteFile, HTTPS_DEFAULT_PORT); + } + + /** + * Prepare uri to retrieve file from xNF. If JWT token was included + * in the queryParameters, it is removed. Other entries are rewritten. + * + * @param scheme scheme which is used during the connection + * @param fileServerData fileServerData including - server address, port, query + * and fragment + * @param remoteFile file which has to be downloaded + * @param defaultPort default port which exchange empty entry for given + * connection type + * @return uri String representing the xNF location + */ + public static String prepareUri(String scheme, FileServerData fileServerData, String remoteFile, int defaultPort) { + int port = fileServerData.port != null ? fileServerData.port : defaultPort; + String query = rewriteQueryWithoutToken(fileServerData.queryParameters); + String fragment = fileServerData.uriRawFragment; + if (!query.isEmpty()) { + query = "?" + query; + } + if (!fragment.isEmpty()) { + fragment = "#" + fragment; + } + return scheme + "://" + fileServerData.serverAddress + ":" + port + remoteFile + query + fragment; + } + + /** + * Returns JWT token string (if single exist) from the queryParameters. + * + * @param fileServerData file server data which contain queryParameters where + * JWT token may exist + * @return JWT token value if single token entry exist or empty string + * elsewhere. + * If JWT token key has no value, empty string will be returned. + */ + public static String getJWTToken(FileServerData fileServerData) { + + if (fileServerData.queryParameters.isEmpty()) { + return ""; + } + boolean jwtTokenKeyPresent = HttpUtils.isQueryWithSingleJWT(fileServerData.queryParameters); + if (!jwtTokenKeyPresent) { + return ""; + } + String token = HttpUtils.getJWTToken(fileServerData.queryParameters); + if (HttpUtils.isBasicAuthDataFilled(fileServerData)) { + logger.warn(HttpUtils.AUTH_JWT_WARN); + } + return token; + } + + /** + * Checks if the queryParameters contains single JWT token entry. Valid + * queryParameters + * contains only one token entry. + * + * @param query queryParameters + * @return true if queryParameters contains single token + */ + public static boolean isQueryWithSingleJWT(List query) { + if (query == null) { + return false; + } + int i = getJWTTokenCount(query); + if (i == 0) { + return false; + } + if (i > 1) { + logger.error(AUTH_JWT_ERROR); + return false; + } + return true; + } + + /** + * Returns the number of JWT token entries. Valid queryParameters contains only + * one token entry. + * + * @param queryElements elements of the queryParameters + * @return true if queryParameters contains single JWT token entry + */ + public static int getJWTTokenCount(List queryElements) { + int i = 0; + for (NameValuePair element : queryElements) { + if (element.getName().equals(JWT_TOKEN_NAME)) { + i++; + } + } + return i; + } + + private static String getJWTToken(List query) { + for (NameValuePair element : query) { + if (!element.getName().equals(JWT_TOKEN_NAME)) { + continue; + } + if (element.getValue() != null) { + return element.getValue(); + } + return ""; + } + return ""; + } + + /** + * Rewrites HTTP queryParameters without JWT token + * + * @param query list of NameValuePair of elements sent in the queryParameters + * @return String representation of queryParameters elements which were provided + * in the input + * Empty string is possible when queryParameters is empty or contains + * only access_token key. + */ + public static String rewriteQueryWithoutToken(List query) { + if (query.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (NameValuePair nvp : query) { + if (nvp.getName().equals(JWT_TOKEN_NAME)) { + continue; + } + sb.append(nvp.getName()); + if (nvp.getValue() != null) { + sb.append("="); + sb.append(nvp.getValue()); + } + sb.append("&"); + } + if ((sb.length() > 0) && (sb.charAt(sb.length() - 1) == '&')) { + sb.deleteCharAt(sb.length() - 1); + } + return sb.toString(); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/CollectAndReportFiles.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/CollectAndReportFiles.java new file mode 100644 index 0000000..c127948 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/CollectAndReportFiles.java @@ -0,0 +1,301 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018, 2020 NOKIA Intellectual Property, 2018-2019 Nordix Foundation. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.tasks; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.StringSerializer; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; +import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig; +import org.onap.dcaegen2.collectors.datafile.datastore.DataStore; +import org.onap.dcaegen2.collectors.datafile.datastore.DataStore.Bucket; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil; +import org.onap.dcaegen2.collectors.datafile.model.Counters; +import org.onap.dcaegen2.collectors.datafile.model.FileData; +import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; +import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; +import reactor.kafka.sender.KafkaSender; +import reactor.kafka.sender.SenderOptions; +import reactor.kafka.sender.SenderRecord; +import reactor.kafka.sender.SenderResult; +import reactor.util.retry.Retry; + +/** + * This implements the main flow of the data file collector. Fetch file ready + * events from the + * message router, fetch new files from the PNF publish these in the data + * router. + */ +@Component +public class CollectAndReportFiles { + + private static Gson gson = new GsonBuilder() // + .disableHtmlEscaping() // + .create(); // + + private static final int NUMBER_OF_WORKER_THREADS = 200; + private static final long FILE_TRANSFER_MAX_RETRIES = 2; + private static final Duration FILE_TRANSFER_INITIAL_RETRY_TIMEOUT = Duration.ofSeconds(2); + + private static final Logger logger = LoggerFactory.getLogger(CollectAndReportFiles.class); + + private final AppConfig appConfig; + + private Counters counters = new Counters(); + + private final KafkaSender kafkaSender; + + private final DataStore dataStore; + + /** + * Constructor for task registration in Datafile Workflow. + * + * @param applicationConfiguration - application configuration + */ + @Autowired + public CollectAndReportFiles(AppConfig applicationConfiguration) { + this.appConfig = applicationConfiguration; + this.kafkaSender = KafkaSender.create(kafkaSenderOptions()); + initCerts(); + + this.dataStore = DataStore.create(applicationConfiguration); + + start(); + } + + private void initCerts() { + try { + CertificateConfig certificateConfig = appConfig.getCertificateConfiguration(); + HttpsClientConnectionManagerUtil.setupOrUpdate(certificateConfig.keyCert, certificateConfig.keyPasswordPath, + certificateConfig.trustedCa, certificateConfig.trustedCaPasswordPath, true); + } catch (DatafileTaskException e) { + logger.error("Could not setup HttpsClient certs, reason: {}", e.getMessage()); + } + } + + /** + * Main function for scheduling for the file collection Workflow. + */ + public void start() { + start(0); + } + + private void start(int delayMillis) { + try { + logger.trace("Starting"); + if (appConfig.isS3Enabled()) { + this.dataStore.create(Bucket.FILES).subscribe(); + this.dataStore.create(Bucket.LOCKS).subscribe(); + } + Thread.sleep(delayMillis); + createMainTask().subscribe(null, s -> start(2000), null); + } catch (Exception e) { + logger.error("Unexpected exception: {}", e.toString(), e); + Thread.currentThread().interrupt(); + } + } + + Flux createMainTask() { + Scheduler scheduler = Schedulers.newParallel("FileCollectorWorker", NUMBER_OF_WORKER_THREADS); + return fetchFromKafka() // + .doOnNext(fileReadyMessage -> counters.threadPoolQueueSize.incrementAndGet()) // + .doOnNext(fileReadyMessage -> counters.incNoOfReceivedEvents()) // + .parallel(NUMBER_OF_WORKER_THREADS) // Each FileReadyMessage in a separate thread + .runOn(scheduler) // + .doOnNext(fileReadyMessage -> counters.threadPoolQueueSize.decrementAndGet()) // + .flatMap(fileReadyMessage -> Flux.fromIterable(FileData.createFileData(fileReadyMessage)), true, 1) // + .flatMap(this::filterNotFetched, false, 1, 1) // + .flatMap(this::fetchFile, false, 1, 1) // + .flatMap(data -> reportFetchedFile(data, this.appConfig.collectedFileTopic), false, 1) // + .sequential() // + .doOnError(t -> logger.error("Received error: {}", t.toString())); // + } + + private Mono deleteLock(FileData info) { + return dataStore.deleteLock(lockName(info.name())).map(b -> info); // + } + + private Mono moveFileToS3Bucket(FilePublishInformation info) { + if (this.appConfig.isS3Enabled()) { + return dataStore.copyFileTo(locaFilePath(info), info.getName()) + .doOnError(t -> logger.warn("Failed to store file '{}' in S3 {}", info.getName(), t.getMessage())) // + .retryWhen(Retry.fixedDelay(10, Duration.ofMillis(1000))) // + .map(f -> info) // + .doOnError(t -> logger.error("Failed to store file '{}' in S3 after retries {}", info.getName(), + t.getMessage())) // + .doOnNext(n -> logger.debug("Stored file in S3: {}", info.getName())) // + .doOnNext(sig -> deleteLocalFile(info)); + } else { + return Mono.just(info); + } + } + + private Mono filterNotFetched(FileData fileData) { + Path localPath = fileData.getLocalFilePath(this.appConfig); + + return dataStore.fileExists(Bucket.FILES, fileData.name()) // + .filter(exists -> !exists) // + .filter(exists -> !localPath.toFile().exists()) // + .map(f -> fileData); // + + } + + private String lockName(String fileName) { + return fileName + ".lck"; + } + + private Path locaFilePath(FilePublishInformation info) { + return Paths.get(this.appConfig.collectedFilesPath, info.getName()); + } + + private void deleteLocalFile(FilePublishInformation info) { + Path path = locaFilePath(info); + try { + Files.delete(path); + } catch (Exception e) { + logger.warn("Could not delete local file: {}, reason:{}", path, e.getMessage()); + } + } + + private Flux reportFetchedFile(FilePublishInformation fileData, String topic) { + String json = gson.toJson(fileData); + return sendDataToStream(topic, fileData.getSourceName(), json) // + .map(result -> fileData); + } + + public Flux> sendDataToStream(String topic, String sourceName, String value) { + return sendDataToKafkaStream(Flux.just(senderRecord(topic, sourceName, value))); + } + + private SenderRecord senderRecord(String topic, String sourceName, String value) { + int correlationMetadata = 2; + String key = null; + var producerRecord = new ProducerRecord<>(topic, null, null, key, value, kafkaHeaders(sourceName)); + return SenderRecord.create(producerRecord, correlationMetadata); + } + + private Iterable
kafkaHeaders(String sourceName) { + ArrayList
result = new ArrayList<>(); + Header h = new RecordHeader("SourceName", sourceName.getBytes()); + result.add(h); + return result; + } + + private Flux> sendDataToKafkaStream(Flux> dataToSend) { + + return kafkaSender.send(dataToSend) // + .doOnError(e -> logger.error("Send to kafka failed", e)); + } + + private SenderOptions kafkaSenderOptions() { + String bootstrapServers = this.appConfig.getKafkaBootStrapServers(); + + Map props = new HashMap<>(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + props.put(ProducerConfig.ACKS_CONFIG, "all"); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + return SenderOptions.create(props); + } + + public Counters getCounters() { + return this.counters; + } + + protected FileCollector createFileCollector() { + return new FileCollector(appConfig, counters); + } + + private Mono fetchFile(FileData fileData) { + return this.dataStore.createLock(lockName(fileData.name())).filter(granted -> granted) // + .map(granted -> createFileCollector()) // + .flatMap(collector -> collector.collectFile(fileData, FILE_TRANSFER_MAX_RETRIES, + FILE_TRANSFER_INITIAL_RETRY_TIMEOUT)) // + .flatMap(this::moveFileToS3Bucket) // + .doOnNext(b -> deleteLock(fileData).subscribe()) // + .doOnError(b -> deleteLock(fileData).subscribe()) // + .onErrorResume(exception -> handleFetchFileFailure(fileData, exception)); // + } + + private Mono handleFetchFileFailure(FileData fileData, Throwable t) { + Path localFilePath = fileData.getLocalFilePath(this.appConfig); + logger.error("File fetching failed, path {}, reason: {}", fileData.remoteFilePath(), t.getMessage()); + deleteFile(localFilePath); + if (Scheme.isFtpScheme(fileData.scheme())) { + counters.incNoOfFailedFtp(); + } else { + counters.incNoOfFailedHttp(); + } + return Mono.empty(); + } + + /** + * Fetch more messages from the message router. This is done in a + * polling/blocking fashion. + */ + private Flux fetchFromKafka() { + KafkaTopicListener listener = new KafkaTopicListener(this.appConfig.getKafkaBootStrapServers(), + this.appConfig.kafkaClientId, this.appConfig.fileReadyEventTopic); + return listener.getFlux() // + .flatMap(this::parseReceivedFileReadyMessage, 1); + + } + + Mono parseReceivedFileReadyMessage(KafkaTopicListener.DataFromTopic data) { + try { + FileReadyMessage msg = gson.fromJson(data.value, FileReadyMessage.class); + logger.debug("Received: {}", msg); + return Mono.just(msg); + } catch (Exception e) { + logger.warn("Could not parse received: {}, reason: {}", data.value, e.getMessage()); + return Mono.empty(); + } + } + + private static void deleteFile(Path localFile) { + logger.trace("Deleting file: {}", localFile); + try { + Files.delete(localFile); + } catch (Exception e) { + logger.trace("Could not delete file: {}, reason: {}", localFile, e.getMessage()); + } + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java new file mode 100644 index 0000000..b6c07e5 --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollector.java @@ -0,0 +1,187 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2020-2022 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.tasks; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Optional; + +import org.apache.commons.io.FileUtils; +import org.onap.dcaegen2.collectors.datafile.commons.FileCollectClient; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; +import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient; +import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient; +import org.onap.dcaegen2.collectors.datafile.ftp.SftpClientSettings; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient; +import org.onap.dcaegen2.collectors.datafile.http.HttpsClientConnectionManagerUtil; +import org.onap.dcaegen2.collectors.datafile.model.Counters; +import org.onap.dcaegen2.collectors.datafile.model.FileData; +import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; +import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; + +/** + * Collects a file from a PNF. + * + * @author Henrik Andersson + */ +public class FileCollector { + + private static final Logger logger = LoggerFactory.getLogger(FileCollector.class); + private final AppConfig appConfig; + private final Counters counters; + + /** + * Constructor. + * + * @param appConfig application configuration + */ + public FileCollector(AppConfig appConfig, Counters counters) { + this.appConfig = appConfig; + this.counters = counters; + } + + /** + * Collects a file from the PNF and stores it in the local file system. + * + * @param fileData data about the file to collect. + * @param numRetries the number of retries if the publishing fails + * @param firstBackoff the time to delay the first retry + * @param contextMap context for logging. + * @return the data needed to publish the file. + */ + public Mono collectFile(FileData fileData, long numRetries, Duration firstBackoff) { + + logger.trace("Entering collectFile with {}", fileData); + + return Mono.just(fileData) // + .cache() // + .flatMap(fd -> tryCollectFile(fileData)) // + .retryWhen(Retry.backoff(numRetries, firstBackoff)) // + .flatMap(FileCollector::checkCollectedFile); + } + + private static Mono checkCollectedFile(Optional info) { + if (info.isPresent()) { + return Mono.just(info.get()); + } else { + // If there is no info, the file is not retrievable + return Mono.error(new DatafileTaskException("Non retryable file transfer failure")); + } + } + + private Mono> tryCollectFile(FileData fileData) { + logger.trace("starting to collectFile {}", fileData.fileInfo.name); + + final String remoteFile = fileData.remoteFilePath(); + final Path localFile = fileData.getLocalFilePath(this.appConfig); + + try (FileCollectClient currentClient = createClient(fileData)) { + currentClient.open(); + FileUtils.forceMkdirParent(localFile.toFile()); + currentClient.collectFile(remoteFile, localFile); + counters.incNoOfCollectedFiles(); + return Mono.just(Optional.of(createFilePublishInformation(fileData))); + } catch (NonRetryableDatafileTaskException nre) { + logger.warn("Failed to download file, not retryable: {} {}, reason: {}", fileData.sourceName(), + fileData.fileInfo.name, nre.getMessage()); + incFailedAttemptsCounter(fileData); + return Mono.just(Optional.empty()); // Give up + } catch (DatafileTaskException e) { + logger.warn("Failed to download file: {} {}, reason: {}", fileData.sourceName(), fileData.fileInfo.name, + e.getMessage()); + incFailedAttemptsCounter(fileData); + return Mono.error(e); + } catch (Exception throwable) { + logger.warn("Failed to close client: {} {}, reason: {}", fileData.sourceName(), fileData.fileInfo.name, + throwable.getMessage(), throwable); + return Mono.just(Optional.of(createFilePublishInformation(fileData))); + } + } + + private void incFailedAttemptsCounter(FileData fileData) { + if (Scheme.isFtpScheme(fileData.scheme())) { + counters.incNoOfFailedFtpAttempts(); + } else { + counters.incNoOfFailedHttpAttempts(); + } + } + + private FileCollectClient createClient(FileData fileData) throws DatafileTaskException { + switch (fileData.scheme()) { + case SFTP: + return createSftpClient(fileData); + case FTPES: + return createFtpesClient(fileData); + case HTTP: + return createHttpClient(fileData); + case HTTPS: + return createHttpsClient(fileData); + default: + throw new DatafileTaskException("Unhandled protocol: " + fileData.scheme()); + } + } + + public FilePublishInformation createFilePublishInformation(FileData fileData) { + FileReadyMessage.MessageMetaData metaData = fileData.messageMetaData; + return FilePublishInformation.builder() // + .productName(metaData.productName()) // + .vendorName(metaData.vendorName()) // + .lastEpochMicrosec(metaData.lastEpochMicrosec) // + .sourceName(metaData.sourceName) // + .startEpochMicrosec(metaData.startEpochMicrosec) // + .timeZoneOffset(metaData.timeZoneOffset) // + .name(metaData.sourceName + "/" + fileData.fileInfo.name) // + .compression(fileData.fileInfo.hashMap.compression) // + .fileFormatType(fileData.fileInfo.hashMap.fileFormatType) // + .fileFormatVersion(fileData.fileInfo.hashMap.fileFormatVersion) // + .changeIdentifier(fileData.messageMetaData.changeIdentifier) // + .objectStoreBucket(this.appConfig.isS3Enabled() ? this.appConfig.getS3Bucket() : null) // + .build(); + } + + protected SftpClient createSftpClient(FileData fileData) { + return new SftpClient(fileData.fileServerData(), new SftpClientSettings(appConfig.getSftpConfiguration())); + } + + protected FtpesClient createFtpesClient(FileData fileData) throws DatafileTaskException { + CertificateConfig config = appConfig.getCertificateConfiguration(); + Path trustedCa = config.trustedCa.isEmpty() ? null : Paths.get(config.trustedCa); + + return new FtpesClient(fileData.fileServerData(), Paths.get(config.keyCert), config.keyPasswordPath, trustedCa, + config.trustedCaPasswordPath); + } + + protected FileCollectClient createHttpClient(FileData fileData) { + return new DfcHttpClient(fileData.fileServerData()); + } + + protected FileCollectClient createHttpsClient(FileData fileData) throws DatafileTaskException { + return new DfcHttpsClient(fileData.fileServerData(), HttpsClientConnectionManagerUtil.instance()); + } +} diff --git a/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/KafkaTopicListener.java b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/KafkaTopicListener.java new file mode 100644 index 0000000..969e5fa --- /dev/null +++ b/datafilecollector/src/main/java/org/onap/dcaegen2/collectors/datafile/tasks/KafkaTopicListener.java @@ -0,0 +1,106 @@ +/*- + * ========================LICENSE_START================================= + * O-RAN-SC + * %% + * Copyright (C) 2021 Nordix Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================LICENSE_END=================================== + */ + +package org.onap.dcaegen2.collectors.datafile.tasks; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import lombok.ToString; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import reactor.core.publisher.Flux; +import reactor.kafka.receiver.KafkaReceiver; +import reactor.kafka.receiver.ReceiverOptions; + +/** + * The class streams incoming requests from a Kafka topic and sends them further + * to a multi cast sink, which several other streams can connect to. + */ +@SuppressWarnings("squid:S2629") // Invoke method(s) only conditionally +public class KafkaTopicListener { + + @ToString + public static class DataFromTopic { + public final String key; + public final String value; + + public DataFromTopic(String key, String value) { + this.key = key; + this.value = value; + } + } + + private static final Logger logger = LoggerFactory.getLogger(KafkaTopicListener.class); + + private final String inputTopic; + private final String kafkaBoostrapServers; + private final String kafkaClientId; + private Flux dataFromTopic; + + public KafkaTopicListener(String kafkaBoostrapServers, String clientId, String topic) { + this.kafkaClientId = clientId; + this.kafkaBoostrapServers = kafkaBoostrapServers; + this.inputTopic = topic; + } + + public Flux getFlux() { + if (this.dataFromTopic == null) { + this.dataFromTopic = startReceiveFromTopic(); + } + return this.dataFromTopic; + } + + private Flux startReceiveFromTopic() { + logger.debug("Listening to kafka topic: {}, client id: {}", this.inputTopic, this.kafkaClientId); + return KafkaReceiver.create(kafkaInputProperties()) // + .receive() // + .doOnNext(input -> logger.debug("Received from kafka topic: {} :{}", this.inputTopic, input.value())) // + .doOnError(t -> logger.error("KafkaTopicReceiver error: {}", t.getMessage())) // + .doFinally(sig -> logger.error("KafkaTopicReceiver stopped, reason: {}", sig)) // + .doFinally(sig -> this.dataFromTopic = null) // + .filter(t -> !t.value().isEmpty() || !t.key().isEmpty()) // + .map(input -> new DataFromTopic(input.key(), input.value())) // + .publish() // + .autoConnect(); + } + + private ReceiverOptions kafkaInputProperties() { + Map consumerProps = new HashMap<>(); + if (this.kafkaBoostrapServers.isEmpty()) { + logger.error("No kafka boostrap server is setup"); + } + consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, this.kafkaBoostrapServers); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "osc-dmaap-adapter-" + inputTopic); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); + consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, this.kafkaClientId); + + return ReceiverOptions.create(consumerProps) + .subscription(Collections.singleton(this.inputTopic)); + } + +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/MockDatafile.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/MockDatafile.java new file mode 100644 index 0000000..16a36c0 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/MockDatafile.java @@ -0,0 +1,341 @@ +/*- + * ========================LICENSE_START================================= + * O-RAN-SC + * %% + * Copyright (C) 2020 Nordix Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================LICENSE_END=================================== + */ + +package org.onap.dcaegen2.collectors.datafile; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; +import org.onap.dcaegen2.collectors.datafile.datastore.DataStore; +import org.onap.dcaegen2.collectors.datafile.datastore.DataStore.Bucket; +import org.onap.dcaegen2.collectors.datafile.model.Counters; +import org.onap.dcaegen2.collectors.datafile.model.FileData; +import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; +import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage; +import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage.MessageMetaData; +import org.onap.dcaegen2.collectors.datafile.tasks.CollectAndReportFiles; +import org.onap.dcaegen2.collectors.datafile.tasks.FileCollector; +import org.onap.dcaegen2.collectors.datafile.tasks.KafkaTopicListener; +import org.onap.dcaegen2.collectors.datafile.tasks.KafkaTopicListener.DataFromTopic; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.TestPropertySource; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +@SpringBootTest(webEnvironment = WebEnvironment.DEFINED_PORT) +@TestPropertySource(properties = { // + "app.ssl.key-store-password-file=./config/ftps_keystore.pass", // + "app.ssl.key-store=./config/ftps_keystore.p12", // + "app.ssl.trust-store-password-file=./config/truststore.pass", // + "app.ssl.trust-store=", // No trust validation + "app.collected-files-path=/tmp/osc_datafile/", // + "logging.file.name=/tmp/datafile.log", // + "spring.main.allow-bean-definition-overriding=true", // + "app.s3.endpointOverride=http://localhost:9000", // + "app.s3.accessKeyId=minio", // + "app.s3.secretAccessKey=miniostorage", // + "app.s3.bucket=ropfiles", // + "app.s3.locksBucket=locks" }) +@SuppressWarnings("squid:S3577") // Not containing any tests since it is a mock. +class MockDatafile { + + private static final int LAST_EPOCH_MICROSEC = 151983; + private static final String SOURCE_NAME = "5GRAN_DU"; + private static final int START_EPOCH_MICROSEC = 15198378; + private static final String TIME_ZONE_OFFSET = "UTC+05:00"; + private static final String PM_FILE_NAME = "PM_FILE_NAME"; + + // This can be any downloadable file on the net + private static final String FTPES_LOCATION = "ftpes:// onap:pano@ftp-ftpes-6:2021/A20000626.2315+0200-2330+0200_GNODEB-15-4.xml.gz"; + private static final String LOCATION = "https://launchpad.net/ubuntu/+source/perf-tools-unstable/1.0+git7ffb3fd-1ubuntu1/+build/13630748/+files/perf-tools-unstable_1.0+git7ffb3fd-1ubuntu1_all.deb"; + private static final String GZIP_COMPRESSION = "gzip"; + private static final String FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec"; + private static final String FILE_FORMAT_VERSION = "V10"; + private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES"; + private static final String CHANGE_TYPE = "FileReady"; + + private static final Logger logger = LoggerFactory.getLogger(MockDatafile.class); + private static Gson gson = new GsonBuilder() // + .disableHtmlEscaping() // + .create(); // + + @LocalServerPort + private int port; + + @Autowired + AppConfig appConfig; + + @Autowired + CollectAndReportFiles scheduledTask; + + private static KafkaReceiver kafkaReceiver; + + private static class KafkaReceiver { + public final String topic; + private DataFromTopic receivedKafkaOutput; + private final Logger logger = LoggerFactory.getLogger(MockDatafile.class); + + int count = 0; + + public KafkaReceiver(AppConfig applicationConfig, String outputTopic) { + this.topic = outputTopic; + + // Create a listener to the output topic. The KafkaTopicListener happens to be + // suitable for that, + + KafkaTopicListener topicListener = new KafkaTopicListener(applicationConfig.getKafkaBootStrapServers(), + "MockDatafile", outputTopic); + + topicListener.getFlux() // + .doOnNext(this::set) // + .doFinally(sig -> logger.info("Finally " + sig)) // + .subscribe(); + } + + private void set(DataFromTopic receivedKafkaOutput) { + this.receivedKafkaOutput = receivedKafkaOutput; + this.count++; + logger.debug("*** received {}, {}", topic, receivedKafkaOutput); + } + + public synchronized String lastKey() { + return this.receivedKafkaOutput.key; + } + + public synchronized String lastValue() { + return this.receivedKafkaOutput.value; + } + + public void reset() { + count = 0; + this.receivedKafkaOutput = new DataFromTopic("", ""); + } + } + + static class FileCollectorMock extends FileCollector { + final AppConfig appConfig; + + public FileCollectorMock(AppConfig appConfig) { + super(appConfig, new Counters()); + this.appConfig = appConfig; + } + + @Override // (override fetchFile to disable the actual file fetching) + public Mono collectFile(FileData fileData, long numRetries, Duration firstBackoff) { + FileCollector fc = new FileCollector(this.appConfig, new Counters()); + FilePublishInformation i = fc.createFilePublishInformation(fileData); + + try { + File from = new File("config/application.yaml"); + File to = new File(this.appConfig.collectedFilesPath + "/" + fileData.name()); + FileUtils.forceMkdirParent(to); + com.google.common.io.Files.copy(from, to); + } catch (Exception e) { + logger.error("Could not copy file {}", e.getMessage()); + } + return Mono.just(i); + } + } + + static class CollectAndReportFilesMock extends CollectAndReportFiles { + final AppConfig appConfig; + + public CollectAndReportFilesMock(AppConfig appConfig) { + super(appConfig); + this.appConfig = appConfig; + } + + @Override // (override fetchFile to disable the actual file fetching) + protected FileCollector createFileCollector() { + return new FileCollectorMock(appConfig); + } + } + + @TestConfiguration + static class TestBeanFactory { + + @Bean + CollectAndReportFiles collectAndReportFiles(@Autowired AppConfig appConfig) { + return new CollectAndReportFilesMock(appConfig); + } + } + + @BeforeEach + void init() { + if (kafkaReceiver == null) { + kafkaReceiver = new KafkaReceiver(this.appConfig, this.appConfig.collectedFileTopic); + } + kafkaReceiver.reset(); + deleteAllFiles(); + } + + @AfterEach + void afterEach() { + DataStore store = DataStore.create(this.appConfig); + store.deleteBucket(Bucket.FILES).block(); + store.deleteBucket(Bucket.LOCKS).block(); + deleteAllFiles(); + + } + + private void deleteAllFiles() { + + try { + FileUtils.deleteDirectory(new File(this.appConfig.collectedFilesPath)); + } catch (IOException e) { + } + } + + @Test + void clear() { + + } + + @Test + void testKafka() throws InterruptedException { + waitForKafkaListener(); + + this.scheduledTask.sendDataToStream(this.appConfig.fileReadyEventTopic, "key", "junk").blockLast(); + + String fileReadyMessage = gson.toJson(fileReadyMessage()); + this.scheduledTask.sendDataToStream(this.appConfig.fileReadyEventTopic, "key", fileReadyMessage).blockLast(); + + await().untilAsserted(() -> assertThat(kafkaReceiver.count).isEqualTo(1)); + String rec = kafkaReceiver.lastValue(); + + assertThat(rec).contains("Ericsson"); + + FilePublishInformation recObj = gson.fromJson(rec, FilePublishInformation.class); + + assertThat(recObj.getName()).isEqualTo(SOURCE_NAME + "/" + PM_FILE_NAME); + } + + @Test + void testS3Concurrency() throws Exception { + waitForKafkaListener(); + + final int NO_OF_OBJECTS = 10; + + Instant startTime = Instant.now(); + + Flux.range(1, NO_OF_OBJECTS) // + .map(i -> gson.toJson(fileReadyMessage("testS3Concurrency_" + i))) // + .flatMap(fileReadyMessage -> scheduledTask.sendDataToStream(appConfig.fileReadyEventTopic, "key", + fileReadyMessage)) // + .blockLast(); // + + while (kafkaReceiver.count < NO_OF_OBJECTS) { + logger.info("sleeping {}", kafkaReceiver.count); + Thread.sleep(1000 * 1); + } + + String rec = kafkaReceiver.lastValue(); + assertThat(rec).contains("Ericsson"); + + final long durationSeconds = Instant.now().getEpochSecond() - startTime.getEpochSecond(); + logger.info("*** Duration :" + durationSeconds + ", objects/second: " + NO_OF_OBJECTS / durationSeconds); + } + + @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests. + private static void waitForKafkaListener() throws InterruptedException { + Thread.sleep(4000); + } + + @Test + @SuppressWarnings("squid:S2699") + void runMock() throws Exception { + logger.warn("**************** Keeping server alive! " + this.port); + synchronized (this) { + this.wait(); + } + } + + FileReadyMessage.Event event(String fileName) { + MessageMetaData messageMetaData = MessageMetaData.builder() // + .lastEpochMicrosec(LAST_EPOCH_MICROSEC) // + .sourceName(SOURCE_NAME) // + .startEpochMicrosec(START_EPOCH_MICROSEC) // + .timeZoneOffset(TIME_ZONE_OFFSET) // + .changeIdentifier(CHANGE_IDENTIFIER) // + .eventName("Noti_RnNode-Ericsson_FileReady").build(); + + FileReadyMessage.FileInfo fileInfo = FileReadyMessage.FileInfo // + .builder() // + .fileFormatType(FILE_FORMAT_TYPE) // + .location(LOCATION) // + .fileFormatVersion(FILE_FORMAT_VERSION) // + .compression(GZIP_COMPRESSION) // + .build(); + + FileReadyMessage.ArrayOfNamedHashMap arrayOfNamedHashMap = FileReadyMessage.ArrayOfNamedHashMap // + .builder() // + .name(fileName) // + .hashMap(fileInfo) // + .build(); + + List arrayOfNamedHashMapList = new ArrayList<>(); + arrayOfNamedHashMapList.add(arrayOfNamedHashMap); + + FileReadyMessage.NotificationFields notificationFields = FileReadyMessage.NotificationFields // + .builder().notificationFieldsVersion("notificationFieldsVersion") // + .changeType(CHANGE_TYPE).changeIdentifier(CHANGE_IDENTIFIER) // + .arrayOfNamedHashMap(arrayOfNamedHashMapList) // + .build(); + + return FileReadyMessage.Event.builder() // + .commonEventHeader(messageMetaData) // + .notificationFields(notificationFields).build(); + } + + private FileReadyMessage fileReadyMessage(String fileName) { + FileReadyMessage message = FileReadyMessage.builder() // + .event(event(fileName)) // + .build(); + return message; + } + + private FileReadyMessage fileReadyMessage() { + return fileReadyMessage(PM_FILE_NAME); + } + +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusControllerTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusControllerTest.java new file mode 100644 index 0000000..8bc6330 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/controllers/StatusControllerTest.java @@ -0,0 +1,73 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. + * Copyright (C) 2020 Nokia. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.controllers; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doReturn; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.model.Counters; +import org.onap.dcaegen2.collectors.datafile.tasks.CollectAndReportFiles; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import reactor.core.publisher.Mono; + +@ExtendWith(MockitoExtension.class) +public class StatusControllerTest { + @Mock + CollectAndReportFiles scheduledTasksMock; + + StatusController controllerUnderTest; + + @BeforeEach + public void setup() { + controllerUnderTest = new StatusController(scheduledTasksMock); + } + + @Test + public void heartbeat_success() { + HttpHeaders httpHeaders = new HttpHeaders(); + + Mono> result = controllerUnderTest.heartbeat(httpHeaders); + + String body = result.block().getBody(); + assertTrue(body.startsWith("I'm living!")); + } + + @Test + public void status() { + Counters counters = new Counters(); + doReturn(counters).when(scheduledTasksMock).getCounters(); + + HttpHeaders httpHeaders = new HttpHeaders(); + + Mono> result = controllerUnderTest.status(httpHeaders); + + String body = result.block().getBody(); + System.out.println(body); + } + +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java new file mode 100644 index 0000000..3423826 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/FtpesClientTest.java @@ -0,0 +1,238 @@ +/* + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2020 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.TrustManager; + +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPSClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatchers; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.springframework.http.HttpStatus; + +public class FtpesClientTest { + + private static final String REMOTE_FILE_PATH = "/dir/sample.txt"; + private static final Path LOCAL_FILE_PATH = Paths.get("target/sample.txt"); + private static final String XNF_ADDRESS = "127.0.0.1"; + private static final int PORT = 8021; + private static final String FTP_KEY_PATH = "ftpKeyPath"; + private static final String FTP_KEY_PASSWORD = "ftpKeyPassword"; + private static final Path TRUSTED_CA_PATH = Paths.get("trustedCaPath"); + private static final String TRUSTED_CA_PASSWORD = "trustedCaPassword"; + + private static final String USERNAME = "bob"; + private static final String PASSWORD = "123"; + + private FTPSClient ftpsClientMock = mock(FTPSClient.class); + private KeyManager keyManagerMock = mock(KeyManager.class); + private TrustManager trustManagerMock = mock(TrustManager.class); + private InputStream inputStreamMock = mock(InputStream.class); + private OutputStream outputStreamMock = mock(OutputStream.class); + + FtpesClient clientUnderTestSpy; + + private FileServerData createFileServerData() { + return FileServerData.builder() // + .serverAddress(XNF_ADDRESS) // + .userId(USERNAME).password(PASSWORD) // + .port(PORT) // + .build(); + } + + @BeforeEach + protected void setUp() throws Exception { + clientUnderTestSpy = spy(new FtpesClient(createFileServerData(), Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD, + TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD)); + clientUnderTestSpy.realFtpsClient = ftpsClientMock; + } + + private void verifyFtpsClientMock_openOk() throws Exception { + doReturn(outputStreamMock).when(clientUnderTestSpy).createOutputStream(LOCAL_FILE_PATH); + + when(ftpsClientMock.retrieveFile(ArgumentMatchers.eq(REMOTE_FILE_PATH), + ArgumentMatchers.any(OutputStream.class))).thenReturn(true); + verify(ftpsClientMock).setNeedClientAuth(true); + verify(ftpsClientMock).setKeyManager(keyManagerMock); + verify(ftpsClientMock).setTrustManager(trustManagerMock); + verify(ftpsClientMock).connect(XNF_ADDRESS, PORT); + verify(ftpsClientMock).login(USERNAME, PASSWORD); + verify(ftpsClientMock).getReplyCode(); + verify(ftpsClientMock, times(1)).enterLocalPassiveMode(); + verify(ftpsClientMock).execPBSZ(0); + verify(ftpsClientMock).execPROT("P"); + verify(ftpsClientMock).setFileType(FTP.BINARY_FILE_TYPE); + verify(ftpsClientMock).setBufferSize(1024 * 1024); + } + + @Test + public void collectFile_allOk() throws Exception { + + doReturn(keyManagerMock).when(clientUnderTestSpy).getKeyManager(Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD); + doReturn(trustManagerMock).when(clientUnderTestSpy).getTrustManager(TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD); + doReturn(outputStreamMock).when(clientUnderTestSpy).createOutputStream(LOCAL_FILE_PATH); + doReturn(true).when(ftpsClientMock).login(USERNAME, PASSWORD); + doReturn(HttpStatus.OK.value()).when(ftpsClientMock).getReplyCode(); + + clientUnderTestSpy.open(); + + doReturn(true).when(ftpsClientMock).retrieveFile(REMOTE_FILE_PATH, outputStreamMock); + clientUnderTestSpy.collectFile(REMOTE_FILE_PATH, LOCAL_FILE_PATH); + + doReturn(true).when(ftpsClientMock).isConnected(); + clientUnderTestSpy.close(); + + verifyFtpsClientMock_openOk(); + verify(ftpsClientMock, times(1)).isConnected(); + verify(ftpsClientMock, times(1)).logout(); + verify(ftpsClientMock, times(1)).disconnect(); + verify(ftpsClientMock, times(1)).retrieveFile(ArgumentMatchers.eq(REMOTE_FILE_PATH), any()); + verifyNoMoreInteractions(ftpsClientMock); + } + + @Test + public void collectFileFaultyOwnKey_shouldFail() throws Exception { + + doReturn(outputStreamMock).when(clientUnderTestSpy).createOutputStream(LOCAL_FILE_PATH); + assertThatThrownBy(() -> clientUnderTestSpy.open()) + .hasMessageContaining("Could not open connection: java.io.FileNotFoundException:"); + + verify(ftpsClientMock).setNeedClientAuth(true); + + doReturn(false).when(ftpsClientMock).isConnected(); + clientUnderTestSpy.close(); + verify(ftpsClientMock).isConnected(); + verifyNoMoreInteractions(ftpsClientMock); + } + + @Test + public void collectFileFaultTrustedCA_shouldFail_no_trustedCA_file() throws Exception { + + doReturn(keyManagerMock).when(clientUnderTestSpy).getKeyManager(Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD); + doThrow(new IOException("problem")).when(clientUnderTestSpy).createInputStream(TRUSTED_CA_PATH); + + assertThatThrownBy(() -> clientUnderTestSpy.open()) + .hasMessage("Could not open connection: java.io.IOException: problem"); + } + + @Test + public void collectFileFaultTrustedCA_shouldFail_empty_trustedCA_file() throws Exception { + + doReturn(keyManagerMock).when(clientUnderTestSpy).getKeyManager(Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD); + doReturn(inputStreamMock).when(clientUnderTestSpy).createInputStream(TRUSTED_CA_PATH); + + assertThatThrownBy(() -> clientUnderTestSpy.open()) + .hasMessage("Could not open connection: java.io.EOFException"); + } + + @Test + public void collectFileFaultyLogin_shouldFail() throws Exception { + + doReturn(keyManagerMock).when(clientUnderTestSpy).getKeyManager(Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD); + doReturn(trustManagerMock).when(clientUnderTestSpy).getTrustManager(TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD); + doReturn(outputStreamMock).when(clientUnderTestSpy).createOutputStream(LOCAL_FILE_PATH); + doReturn(false).when(ftpsClientMock).login(USERNAME, PASSWORD); + + assertThatThrownBy(() -> clientUnderTestSpy.open()).hasMessage("Unable to log in to xNF. 127.0.0.1"); + + verify(ftpsClientMock).setNeedClientAuth(true); + verify(ftpsClientMock).setKeyManager(keyManagerMock); + verify(ftpsClientMock).setTrustManager(trustManagerMock); + verify(ftpsClientMock).connect(XNF_ADDRESS, PORT); + verify(ftpsClientMock).login(USERNAME, PASSWORD); + } + + @Test + public void collectFileBadRequestResponse_shouldFail() throws Exception { + doReturn(keyManagerMock).when(clientUnderTestSpy).getKeyManager(Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD); + doReturn(trustManagerMock).when(clientUnderTestSpy).getTrustManager(TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD); + doReturn(outputStreamMock).when(clientUnderTestSpy).createOutputStream(LOCAL_FILE_PATH); + doReturn(true).when(ftpsClientMock).login(USERNAME, PASSWORD); + doReturn(503).when(ftpsClientMock).getReplyCode(); + + assertThatThrownBy(() -> clientUnderTestSpy.open()) + .hasMessage("Unable to connect to xNF. 127.0.0.1 xNF reply code: 503"); + + verify(ftpsClientMock).setNeedClientAuth(true); + verify(ftpsClientMock).setKeyManager(keyManagerMock); + verify(ftpsClientMock).setTrustManager(trustManagerMock); + verify(ftpsClientMock).connect(XNF_ADDRESS, PORT); + verify(ftpsClientMock).login(USERNAME, PASSWORD); + verify(ftpsClientMock, times(2)).getReplyCode(); + verifyNoMoreInteractions(ftpsClientMock); + } + + @Test + public void collectFile_shouldFail() throws Exception { + doReturn(keyManagerMock).when(clientUnderTestSpy).getKeyManager(Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD); + doReturn(trustManagerMock).when(clientUnderTestSpy).getTrustManager(TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD); + doReturn(outputStreamMock).when(clientUnderTestSpy).createOutputStream(LOCAL_FILE_PATH); + doReturn(true).when(ftpsClientMock).login(USERNAME, PASSWORD); + doReturn(HttpStatus.OK.value()).when(ftpsClientMock).getReplyCode(); + clientUnderTestSpy.open(); + + doReturn(false).when(ftpsClientMock).retrieveFile(REMOTE_FILE_PATH, outputStreamMock); + + assertThatThrownBy(() -> clientUnderTestSpy.collectFile(REMOTE_FILE_PATH, LOCAL_FILE_PATH)) + .hasMessageContaining(REMOTE_FILE_PATH).hasMessageContaining("No retry"); + + verifyFtpsClientMock_openOk(); + verify(ftpsClientMock, times(1)).retrieveFile(ArgumentMatchers.eq(REMOTE_FILE_PATH), any()); + verifyNoMoreInteractions(ftpsClientMock); + } + + @Test + public void collectFile_shouldFail_ioexception() throws Exception { + doReturn(keyManagerMock).when(clientUnderTestSpy).getKeyManager(Paths.get(FTP_KEY_PATH), FTP_KEY_PASSWORD); + doReturn(trustManagerMock).when(clientUnderTestSpy).getTrustManager(TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD); + doReturn(outputStreamMock).when(clientUnderTestSpy).createOutputStream(LOCAL_FILE_PATH); + doReturn(true).when(ftpsClientMock).login(USERNAME, PASSWORD); + doReturn(HttpStatus.OK.value()).when(ftpsClientMock).getReplyCode(); + clientUnderTestSpy.open(); + when(ftpsClientMock.isConnected()).thenReturn(false); + + doThrow(new IOException("problem")).when(ftpsClientMock).retrieveFile(REMOTE_FILE_PATH, outputStreamMock); + + assertThatThrownBy(() -> clientUnderTestSpy.collectFile(REMOTE_FILE_PATH, LOCAL_FILE_PATH)) + .hasMessage("Could not fetch file: java.io.IOException: problem"); + + verifyFtpsClientMock_openOk(); + verify(ftpsClientMock, times(1)).retrieveFile(ArgumentMatchers.eq(REMOTE_FILE_PATH), any()); + verifyNoMoreInteractions(ftpsClientMock); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettingsTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettingsTest.java new file mode 100644 index 0000000..5ee379b --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientSettingsTest.java @@ -0,0 +1,67 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2020 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.onap.dcaegen2.collectors.datafile.configuration.SftpConfig; + +public class SftpClientSettingsTest { + + @Test + public void shouldUseFtpStrictHostChecking(@TempDir Path tempDir) throws Exception { + File knowHostsFile = new File(tempDir.toFile(), "known_hosts"); + knowHostsFile.createNewFile(); + + SftpConfig config = createSampleSftpConfigWithStrictHostChecking(knowHostsFile.getAbsolutePath()); + SftpClientSettings sftpClient = new SftpClientSettings(config); + + assertThat(sftpClient.shouldUseStrictHostChecking()).isTrue(); + } + + @Test + public void shouldNotUseFtpStrictHostChecking_whenFileDoesNotExist() { + SftpConfig config = createSampleSftpConfigWithStrictHostChecking("unknown_file"); + SftpClientSettings sftpClient = new SftpClientSettings(config); + + sftpClient.shouldUseStrictHostChecking(); + assertThat(sftpClient.shouldUseStrictHostChecking()).isFalse(); + } + + @Test + public void shouldNotUseFtpStrictHostChecking_whenExplicitlySwitchedOff() { + SftpClientSettings sftpClient = new SftpClientSettings(createSampleSftpConfigNoStrictHostChecking()); + sftpClient.shouldUseStrictHostChecking(); + assertThat(sftpClient.shouldUseStrictHostChecking()).isFalse(); + } + + private SftpConfig createSampleSftpConfigNoStrictHostChecking() { + return SftpConfig.builder() // + .strictHostKeyChecking(false).knownHostsFilePath("N/A").build(); + } + + private SftpConfig createSampleSftpConfigWithStrictHostChecking(String pathToKnownHostsFile) { + return SftpConfig.builder() // + .strictHostKeyChecking(true).knownHostsFilePath(pathToKnownHostsFile).build(); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java new file mode 100644 index 0000000..596bec8 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/ftp/SftpClientTest.java @@ -0,0 +1,237 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2020 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.ftp; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpException; + +import java.nio.file.Paths; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.configuration.SftpConfig; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; + +@ExtendWith(MockitoExtension.class) +public class SftpClientTest { + + private static final String HOST = "127.0.0.1"; + private static final int SFTP_PORT = 1021; + private static final String USERNAME = "bob"; + private static final String PASSWORD = "123"; + + @Mock + private JSch jschMock; + + @Mock + private Session sessionMock; + + @Mock + private ChannelSftp channelMock; + + @Test + public void openWithPort_success() throws Exception { + FileServerData expectedFileServerData = FileServerData.builder() // + .serverAddress(HOST) // + .userId(USERNAME) // + .password(PASSWORD) // + .port(SFTP_PORT) // + .build(); + + SftpClient sftpClientSpy = spy(new SftpClient(expectedFileServerData, createSampleSftpClientSettings())); + + doReturn(jschMock).when(sftpClientSpy).createJsch(); + when(jschMock.getSession(anyString(), anyString(), anyInt())).thenReturn(sessionMock); + when(sessionMock.openChannel(anyString())).thenReturn(channelMock); + + sftpClientSpy.open(); + + verify(jschMock).getSession(USERNAME, HOST, SFTP_PORT); + verify(sessionMock).setConfig("StrictHostKeyChecking", "no"); + verify(sessionMock).setPassword(PASSWORD); + verify(sessionMock).connect(); + verify(sessionMock).openChannel("sftp"); + verifyNoMoreInteractions(sessionMock); + + verify(channelMock).connect(); + verifyNoMoreInteractions(channelMock); + } + + @Test + public void openWithoutPort_success() throws Exception { + FileServerData expectedFileServerData = FileServerData.builder() // + .serverAddress(HOST) // + .userId(USERNAME) // + .password(PASSWORD) // + .port(null) // + .build(); + + SftpClient sftpClientSpy = spy(new SftpClient(expectedFileServerData, createSampleSftpClientSettings())); + + doReturn(jschMock).when(sftpClientSpy).createJsch(); + when(jschMock.getSession(anyString(), anyString(), anyInt())).thenReturn(sessionMock); + when(sessionMock.openChannel(anyString())).thenReturn(channelMock); + + sftpClientSpy.open(); + + verify(jschMock).getSession(USERNAME, HOST, 22); + } + + @Test + public void open_throwsExceptionWithRetry() throws Exception { + FileServerData expectedFileServerData = FileServerData.builder() // + .serverAddress(HOST) // + .userId(USERNAME) // + .password(PASSWORD) // + .port(SFTP_PORT) // + .build(); + + SftpClient sftpClientSpy = spy(new SftpClient(expectedFileServerData, createSampleSftpClientSettings())); + + doReturn(jschMock).when(sftpClientSpy).createJsch(); + when(jschMock.getSession(anyString(), anyString(), anyInt())).thenThrow(new JSchException("Failed")); + + DatafileTaskException exception = assertThrows(DatafileTaskException.class, () -> sftpClientSpy.open()); + assertEquals("Could not open Sftp client. com.jcraft.jsch.JSchException: Failed", exception.getMessage()); + } + + @Test + public void openAuthFail_throwsExceptionWithoutRetry() throws Exception { + FileServerData expectedFileServerData = FileServerData.builder() // + .serverAddress(HOST) // + .userId(USERNAME) // + .password(PASSWORD) // + .port(SFTP_PORT) // + .build(); + + SftpClient sftpClientSpy = spy(new SftpClient(expectedFileServerData, createSampleSftpClientSettings())); + + doReturn(jschMock).when(sftpClientSpy).createJsch(); + when(jschMock.getSession(anyString(), anyString(), anyInt())).thenThrow(new JSchException("Auth fail")); + + NonRetryableDatafileTaskException exception = + assertThrows(NonRetryableDatafileTaskException.class, () -> sftpClientSpy.open()); + assertEquals( + "Could not open Sftp client, no retry attempts will be done. com.jcraft.jsch.JSchException: Auth fail", + exception.getMessage()); + } + + @SuppressWarnings("resource") + @Test + public void collectFile_success() throws DatafileTaskException, SftpException { + FileServerData expectedFileServerData = FileServerData.builder() // + .serverAddress(HOST) // + .userId(USERNAME) // + .password(PASSWORD) // + .port(SFTP_PORT) // + .build(); + SftpClient sftpClient = new SftpClient(expectedFileServerData, createSampleSftpClientSettings()); + + sftpClient.sftpChannel = channelMock; + + sftpClient.collectFile("remote.xml", Paths.get("local.xml")); + + verify(channelMock).get("remote.xml", "local.xml"); + verifyNoMoreInteractions(channelMock); + } + + @Test + public void collectFile_throwsExceptionWithRetry() throws SftpException { + FileServerData expectedFileServerData = FileServerData.builder() // + .serverAddress(HOST) // + .userId(USERNAME) // + .password(PASSWORD) // + .port(SFTP_PORT) // + .build(); + + try (SftpClient sftpClient = new SftpClient(expectedFileServerData, createSampleSftpClientSettings())) { + sftpClient.sftpChannel = channelMock; + doThrow(new SftpException(ChannelSftp.SSH_FX_BAD_MESSAGE, "Failed")).when(channelMock).get(anyString(), + anyString()); + + assertThatThrownBy(() -> sftpClient.collectFile("remoteFile", Paths.get("localFile"))) + .isInstanceOf(DatafileTaskException.class).hasMessageStartingWith("Unable to get file from xNF. ") + .hasMessageContaining(HOST); + } + } + + @Test + public void collectFileFileMissing_throwsExceptionWithoutRetry() throws SftpException { + FileServerData expectedFileServerData = FileServerData.builder() // + .serverAddress(HOST) // + .userId(USERNAME) // + .password(PASSWORD) // + .port(SFTP_PORT) // + .build(); + + try (SftpClient sftpClient = new SftpClient(expectedFileServerData, createSampleSftpClientSettings())) { + sftpClient.sftpChannel = channelMock; + doThrow(new SftpException(ChannelSftp.SSH_FX_NO_SUCH_FILE, "Failed")).when(channelMock).get(anyString(), + anyString()); + + assertThatThrownBy(() -> sftpClient.collectFile("remoteFile", Paths.get("localFile"))) + .isInstanceOf(NonRetryableDatafileTaskException.class) + .hasMessageStartingWith("Unable to get file from xNF. No retry attempts will be done") + .hasMessageContaining("" + SFTP_PORT); + } + } + + @Test + public void close_success() { + SftpClient sftpClient = new SftpClient(null, createSampleSftpClientSettings()); + + sftpClient.session = sessionMock; + sftpClient.sftpChannel = channelMock; + + sftpClient.close(); + + verify(sessionMock).disconnect(); + verifyNoMoreInteractions(sessionMock); + + verify(channelMock).exit();; + verifyNoMoreInteractions(channelMock); + } + + private SftpClientSettings createSampleSftpClientSettings() { + return new SftpClientSettings(createSampleSftpConfigNoStrictHostChecking()); + } + + private SftpConfig createSampleSftpConfigNoStrictHostChecking() { + return SftpConfig.builder() // + .strictHostKeyChecking(false).knownHostsFilePath("N/A").build(); + } + +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java new file mode 100644 index 0000000..8550644 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpClientTest.java @@ -0,0 +1,159 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2020-2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.nio.file.Path; + +import org.apache.hc.core5.net.URIBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.service.HttpUtils; + +import reactor.core.publisher.Flux; +import reactor.netty.http.client.HttpClientConfig; + +@ExtendWith(MockitoExtension.class) +class DfcHttpClientTest { + + private static final String USERNAME = "bob"; + private static final String PASSWORD = "123"; + private static final String XNF_ADDRESS = "127.0.0.1"; + private static final int PORT = 80; + private static final String JWT_PASSWORD = "thisIsThePassword"; + private static String ACCESS_TOKEN = "access_token"; + + @Mock + private Path pathMock; + + DfcHttpClient dfcHttpClientSpy; + + @BeforeEach + public void setup() { + dfcHttpClientSpy = spy(new DfcHttpClient(createFileServerData())); + } + + @Test + void openConnection_successBasicAuthSetup() throws DatafileTaskException { + dfcHttpClientSpy.open(); + HttpClientConfig config = dfcHttpClientSpy.client.configuration(); + assertEquals(HttpUtils.basicAuthContent(USERNAME, PASSWORD), config.headers().get("Authorization")); + } + + @Test + void openConnection_failedBasicAuthSetupThrowException() { + FileServerData serverData = + FileServerData.builder().serverAddress(XNF_ADDRESS).userId(USERNAME).password("").port(PORT).build(); + + DfcHttpClient dfcHttpClientSpy = spy(new DfcHttpClient(serverData)); + + assertThatThrownBy(() -> dfcHttpClientSpy.open()) + .hasMessageContaining("Not sufficient basic auth data for file."); + } + + @Test + void collectFile_AllOk() throws Exception { + String REMOTE_FILE = "any"; + Flux fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes())); + + dfcHttpClientSpy.open(); + + when(dfcHttpClientSpy.getServerResponse(any())).thenReturn(fis); + doReturn(false).when(dfcHttpClientSpy).isDownloadFailed(any()); + + dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock); + dfcHttpClientSpy.close(); + + verify(dfcHttpClientSpy, times(1)).getServerResponse(REMOTE_FILE); + verify(dfcHttpClientSpy, times(1)).processDataFromServer(any(), any(), any()); + verify(dfcHttpClientSpy, times(1)).isDownloadFailed(any()); + } + + @Test + void collectFile_AllOkWithJWTToken() throws Exception { + dfcHttpClientSpy = spy(new DfcHttpClient(fileServerDataWithJWTToken())); + String REMOTE_FILE = "any"; + Flux fis = Flux.just(new ByteArrayInputStream("ReturnedString".getBytes())); + + dfcHttpClientSpy.open(); + HttpClientConfig config = dfcHttpClientSpy.client.configuration(); + assertEquals(HttpUtils.jwtAuthContent(JWT_PASSWORD), config.headers().get("Authorization")); + + when(dfcHttpClientSpy.getServerResponse(any())).thenReturn(fis); + doReturn(false).when(dfcHttpClientSpy).isDownloadFailed(any()); + + dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock); + dfcHttpClientSpy.close(); + + verify(dfcHttpClientSpy, times(1)).getServerResponse(ArgumentMatchers.eq(REMOTE_FILE)); + verify(dfcHttpClientSpy, times(1)).processDataFromServer(any(), any(), any()); + verify(dfcHttpClientSpy, times(1)).isDownloadFailed(any()); + } + + @Test + void collectFile_No200ResponseWriteToErrorMessage() throws DatafileTaskException { + String ERROR_RESPONSE = "This is unexpected message"; + String REMOTE_FILE = "any"; + Flux fis = Flux.error(new Throwable(ERROR_RESPONSE)); + + dfcHttpClientSpy.open(); + + doReturn(fis).when(dfcHttpClientSpy).getServerResponse(any()); + + assertThatThrownBy(() -> dfcHttpClientSpy.collectFile(REMOTE_FILE, pathMock)) + .hasMessageContaining(ERROR_RESPONSE); + verify(dfcHttpClientSpy, times(1)).getServerResponse(REMOTE_FILE); + verify(dfcHttpClientSpy, times(1)).processFailedConnectionWithServer(any(), any()); + dfcHttpClientSpy.close(); + } + + @Test + void isResponseOk_validateResponse() { + assertTrue(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.NETTY_RESPONSE_OK)); + assertFalse(dfcHttpClientSpy.isResponseOk(HttpClientResponseHelper.RESPONSE_ANY_NO_OK)); + } + + private FileServerData createFileServerData() { + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId(USERNAME).password(PASSWORD).port(PORT) + .build(); + } + + private FileServerData fileServerDataWithJWTToken() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId("").password("").port(PORT) + .queryParameters(new URIBuilder(query).getQueryParams()).build(); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java new file mode 100644 index 0000000..4295fe8 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/DfcHttpsClientTest.java @@ -0,0 +1,178 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.nio.file.Path; + +import org.apache.hc.core5.net.URIBuilder; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; + +@ExtendWith(MockitoExtension.class) +class DfcHttpsClientTest { + + private static final String USERNAME = "bob"; + private static final String PASSWORD = "123"; + private static final String XNF_ADDRESS = "127.0.0.1"; + private static final int PORT = 443; + private static final String JWT_PASSWORD = "thisIsThePassword"; + private static String ACCESS_TOKEN = "access_token"; + private static String remoteFile = "remoteFile"; + + @Mock + private PoolingHttpClientConnectionManager connectionManager; + @Mock + private Path localFile; + + DfcHttpsClient dfcHttpsClientSpy; + + @BeforeEach + public void setup() { + dfcHttpsClientSpy = spy(new DfcHttpsClient(createFileServerData(), connectionManager)); + } + + @Test + void fileServerData_properLocationBasicAuth() throws Exception { + boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow(); + assertEquals(true, result); + } + + @Test + void fileServerData_properLocationNoBasicAuth() throws Exception { + dfcHttpsClientSpy = spy(new DfcHttpsClient(emptyUserInFileServerData(), connectionManager)); + + boolean result = dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow(); + assertEquals(false, result); + } + + @Test + void fileServerData_improperAuthDataExceptionOccurred() throws Exception { + dfcHttpsClientSpy = spy(new DfcHttpsClient(invalidUserInFileServerData(), connectionManager)); + + assertThrows(DatafileTaskException.class, () -> dfcHttpsClientSpy.basicAuthValidNotPresentOrThrow()); + } + + @Test + void dfcHttpsClient_flow_successfulCallAndResponseProcessing() throws Exception { + doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + doReturn((long) 3).when(dfcHttpsClientSpy).writeFile(eq(localFile), any(InputStream.class)); + + dfcHttpsClientSpy.open(); + dfcHttpsClientSpy.collectFile(remoteFile, localFile); + dfcHttpsClientSpy.close(); + + verify(dfcHttpsClientSpy, times(1)).makeCall(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)).executeHttpClient(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)).processResponse(HttpClientResponseHelper.APACHE_RESPONSE_OK, localFile); + verify(dfcHttpsClientSpy, times(1)).writeFile(eq(localFile), any(InputStream.class)); + } + + @Test + void dfcHttpsClient_flow_successfulCallWithJWTAndResponseProcessing() throws Exception { + FileServerData serverData = jWTTokenInFileServerData(); + dfcHttpsClientSpy = spy(new DfcHttpsClient(serverData, connectionManager)); + + doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + doReturn((long) 3).when(dfcHttpsClientSpy).writeFile(eq(localFile), any(InputStream.class)); + + dfcHttpsClientSpy.open(); + dfcHttpsClientSpy.collectFile(remoteFile, localFile); + dfcHttpsClientSpy.close(); + + verify(dfcHttpsClientSpy, times(1)).makeCall(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)).executeHttpClient(any(HttpGet.class)); + verify(dfcHttpsClientSpy, times(1)).processResponse(HttpClientResponseHelper.APACHE_RESPONSE_OK, localFile); + verify(dfcHttpsClientSpy, times(1)).writeFile(eq(localFile), any(InputStream.class)); + String str = serverData.toString(); + assertFalse(str.contains(JWT_PASSWORD)); + } + + @Test + void dfcHttpsClient_flow_failedCallUnexpectedResponseCode() throws Exception { + doReturn(HttpClientResponseHelper.APACHE_RESPONSE_OK).when(dfcHttpsClientSpy) + .executeHttpClient(any(HttpGet.class)); + doReturn(false).when(dfcHttpsClientSpy).isResponseOk(any(HttpResponse.class)); + + dfcHttpsClientSpy.open(); + + assertThrows(DatafileTaskException.class, () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile)); + } + + @Test + void dfcHttpsClient_flow_failedCallConnectionTimeout() throws Exception { + doThrow(ConnectTimeoutException.class).when(dfcHttpsClientSpy).executeHttpClient(any(HttpGet.class)); + + dfcHttpsClientSpy.open(); + + assertThrows(NonRetryableDatafileTaskException.class, + () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile)); + } + + @Test + void dfcHttpsClient_flow_failedCallIOExceptionForExecuteHttpClient() throws Exception { + doThrow(IOException.class).when(dfcHttpsClientSpy).executeHttpClient(any(HttpGet.class)); + + dfcHttpsClientSpy.open(); + + assertThrows(DatafileTaskException.class, () -> dfcHttpsClientSpy.collectFile(remoteFile, localFile)); + } + + private FileServerData createFileServerData() { + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId(USERNAME).password(PASSWORD).port(PORT) + .build(); + } + + private FileServerData emptyUserInFileServerData() { + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId("").password("").port(PORT).build(); + } + + private FileServerData invalidUserInFileServerData() { + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId(USERNAME).password("").port(PORT).build(); + } + + private FileServerData jWTTokenInFileServerData() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId("").password("").port(PORT) + .queryParameters(new URIBuilder(query).getQueryParams()).build(); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java new file mode 100644 index 0000000..3df2cad --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpClientResponseHelper.java @@ -0,0 +1,419 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2020-2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.cookie.Cookie; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.http.Header; +import org.apache.http.HeaderIterator; +import org.apache.http.HttpEntity; +import org.apache.http.ProtocolVersion; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.params.HttpParams; + +import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClientResponse; +import reactor.util.context.Context; +import reactor.util.context.ContextView; + +public class HttpClientResponseHelper { + + public static final HttpClientResponse NETTY_RESPONSE_OK = new HttpClientResponse() { + + @Override + public Map> cookies() { + return null; + } + + @Override + public boolean isKeepAlive() { + return false; + } + + @Override + public boolean isWebsocket() { + return false; + } + + @Override + public HttpMethod method() { + return null; + } + + @Override + public String path() { + return null; + } + + @Override + public String fullPath() { + return null; + } + + @Override + public String requestId() { + return null; + } + + @Override + public String uri() { + return null; + } + + @Override + public HttpVersion version() { + return null; + } + + @Override + public Context currentContext() { + return null; + } + + @Override + public ContextView currentContextView() { + return null; + } + + @Override + public String[] redirectedFrom() { + return new String[0]; + } + + @Override + public HttpHeaders requestHeaders() { + return null; + } + + @Override + public String resourceUrl() { + return null; + } + + @Override + public HttpHeaders responseHeaders() { + return null; + } + + @Override + public HttpResponseStatus status() { + return HttpResponseStatus.OK; + } + + @Override + public Mono trailerHeaders() { + return null; + } + }; + + public static final HttpClientResponse RESPONSE_ANY_NO_OK = new HttpClientResponse() { + + @Override + public Map> cookies() { + return null; + } + + @Override + public boolean isKeepAlive() { + return false; + } + + @Override + public boolean isWebsocket() { + return false; + } + + @Override + public HttpMethod method() { + return null; + } + + @Override + public String fullPath() { + return null; + } + + @Override + public String requestId() { + return null; + } + + @Override + public String uri() { + return null; + } + + @Override + public HttpVersion version() { + return null; + } + + @Override + public Context currentContext() { + return null; + } + + @Override + public ContextView currentContextView() { + return null; + } + + @Override + public String[] redirectedFrom() { + return new String[0]; + } + + @Override + public HttpHeaders requestHeaders() { + return null; + } + + @Override + public String resourceUrl() { + return null; + } + + @Override + public HttpHeaders responseHeaders() { + return null; + } + + @Override + public HttpResponseStatus status() { + return HttpResponseStatus.NOT_IMPLEMENTED; + } + + @Override + public Mono trailerHeaders() { + return null; + } + }; + + public static final CloseableHttpResponse APACHE_RESPONSE_OK = new CloseableHttpResponse() { + @Override + public void close() throws IOException { + getEntity().getContent().close(); + } + + @Override + public StatusLine getStatusLine() { + return new StatusLine() { + @Override + public ProtocolVersion getProtocolVersion() { + return null; + } + + @Override + public int getStatusCode() { + return 200; + } + + @Override + public String getReasonPhrase() { + return null; + } + }; + } + + @Override + public void setStatusLine(StatusLine statusLine) { + + } + + @Override + public void setStatusLine(ProtocolVersion protocolVersion, int i) { + + } + + @Override + public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) { + + } + + @Override + public void setStatusCode(int i) throws IllegalStateException { + + } + + @Override + public void setReasonPhrase(String s) throws IllegalStateException { + + } + + @Override + public HttpEntity getEntity() { + return new HttpEntity() { + @Override + public boolean isRepeatable() { + return false; + } + + @Override + public boolean isChunked() { + return false; + } + + @Override + public long getContentLength() { + return 0; + } + + @Override + public Header getContentType() { + return null; + } + + @Override + public Header getContentEncoding() { + return null; + } + + @Override + public InputStream getContent() throws IOException, UnsupportedOperationException { + return new ByteArrayInputStream("abc".getBytes()); + } + + @Override + public void writeTo(OutputStream outputStream) throws IOException { + + } + + @Override + public boolean isStreaming() { + return false; + } + + @Override + public void consumeContent() throws IOException { + + } + }; + } + + @Override + public void setEntity(HttpEntity httpEntity) { + + } + + @Override + public Locale getLocale() { + return null; + } + + @Override + public void setLocale(Locale locale) { + + } + + @Override + public ProtocolVersion getProtocolVersion() { + return null; + } + + @Override + public boolean containsHeader(String s) { + return false; + } + + @Override + public Header[] getHeaders(String s) { + return new Header[0]; + } + + @Override + public Header getFirstHeader(String s) { + return null; + } + + @Override + public Header getLastHeader(String s) { + return null; + } + + @Override + public Header[] getAllHeaders() { + return new Header[0]; + } + + @Override + public void addHeader(Header header) { + + } + + @Override + public void addHeader(String s, String s1) { + + } + + @Override + public void setHeader(Header header) { + + } + + @Override + public void setHeader(String s, String s1) { + + } + + @Override + public void setHeaders(Header[] headers) { + + } + + @Override + public void removeHeader(Header header) { + + } + + @Override + public void removeHeaders(String s) { + + } + + @Override + public HeaderIterator headerIterator() { + return null; + } + + @Override + public HeaderIterator headerIterator(String s) { + return null; + } + + @Override + public HttpParams getParams() { + return null; + } + + @Override + public void setParams(HttpParams params) { + } + + }; +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java new file mode 100644 index 0000000..bb1a93f --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/http/HttpsClientConnectionManagerUtilTest.java @@ -0,0 +1,54 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ +package org.onap.dcaegen2.collectors.datafile.http; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; + +@ExtendWith(MockitoExtension.class) +public class HttpsClientConnectionManagerUtilTest { + + private static final String KEY_PATH = "src/test/resources/keystore.p12"; + private static final String KEY_PASSWORD = "src/test/resources/keystore.pass"; + private static final String KEY_IMPROPER_PASSWORD = "src/test/resources/dfc.jks.pass"; + private static final String TRUSTED_CA_PATH = "src/test/resources/trust.jks"; + private static final String TRUSTED_CA_PASSWORD = "src/test/resources/trust.pass"; + + @Test + public void emptyManager_shouldThrowException() { + assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance()); + } + + @Test + public void creatingManager_successfulCase() throws Exception { + HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, KEY_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD, // + true); + assertNotNull(HttpsClientConnectionManagerUtil.instance()); + } + + @Test + public void creatingManager_improperSecretShouldThrowException() { + assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.setupOrUpdate(KEY_PATH, // + KEY_IMPROPER_PASSWORD, TRUSTED_CA_PATH, TRUSTED_CA_PASSWORD, true)); + assertThrows(DatafileTaskException.class, () -> HttpsClientConnectionManagerUtil.instance()); + } + +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java new file mode 100644 index 0000000..413cd13 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/scheme/SchemeTest.java @@ -0,0 +1,48 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2020 Nokia. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.scheme; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; +import org.onap.dcaegen2.collectors.datafile.commons.Scheme; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; + +public class SchemeTest { + + @Test + public void shouldReturnSchemeForSupportedProtocol() throws DatafileTaskException { + assertEquals(Scheme.FTPES, Scheme.getSchemeFromString("FTPES")); + assertEquals(Scheme.SFTP, Scheme.getSchemeFromString("SFTP")); + assertEquals(Scheme.HTTP, Scheme.getSchemeFromString("HTTP")); + assertEquals(Scheme.HTTPS, Scheme.getSchemeFromString("HTTPS")); + } + + @Test + public void shouldThrowExceptionForUnsupportedProtocol() { + assertThrows(DatafileTaskException.class, () -> Scheme.getSchemeFromString("FTPS")); + } + + @Test + public void shouldThrowExceptionForInvalidProtocol() { + assertThrows(DatafileTaskException.class, () -> Scheme.getSchemeFromString("invalid")); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java new file mode 100644 index 0000000..0ee9f72 --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/service/HttpUtilsTest.java @@ -0,0 +1,159 @@ +/* + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Modifications Copyright (C) 2021 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URISyntaxException; +import java.util.List; + +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.net.URIBuilder; +import org.junit.jupiter.api.Test; +import org.onap.dcaegen2.collectors.datafile.commons.FileServerData; + +class HttpUtilsTest { + + private static final String XNF_ADDRESS = "127.0.0.1"; + private static final int PORT = 443; + private static final String JWT_PASSWORD = "thisIsThePassword"; + private static final String ACCESS_TOKEN = "access_token"; + private static final String ANOTHER_TOKEN = "another_token"; + private static final String ANOTHER_DATA = "another_data"; + private static final String FRAGMENT = "thisIsTheFragment"; + private static final String USERNAME = "bob"; + private static final String PASSWORD = "123"; + + @Test + void shouldReturnSuccessfulResponse() { + assertTrue(HttpUtils.isSuccessfulResponseCodeWithDataRouter(200)); + } + + @Test + void shouldReturnBadResponse() { + assertFalse(HttpUtils.isSuccessfulResponseCodeWithDataRouter(404)); + } + + @Test + void isSingleQueryWithJWT_validToken() throws URISyntaxException { + assertTrue(HttpUtils.isQueryWithSingleJWT(validTokenSingleQueryData())); + assertTrue(HttpUtils.isQueryWithSingleJWT(validTokenDoubleQueryData())); + } + + @Test + void isSingleQueryWithJWT_invalidToken() throws URISyntaxException { + assertFalse(HttpUtils.isQueryWithSingleJWT(validQueryNoToken())); + assertFalse(HttpUtils.isQueryWithSingleJWT(queryDataDoubleToken())); + assertFalse(HttpUtils.isQueryWithSingleJWT(null)); + } + + @Test + void getJWTToken_jWTTokenPresent() throws URISyntaxException { + assertEquals(JWT_PASSWORD, HttpUtils.getJWTToken(fileServerDataWithJWTToken())); + assertEquals(JWT_PASSWORD, HttpUtils.getJWTToken(fileServerDataWithJWTTokenLongQueryAndFragment())); + } + + @Test + void getJWTToken_JWTTokenNotPresent() throws URISyntaxException { + assertEquals("", HttpUtils.getJWTToken(fileServerDataQueryWithoutToken())); + } + + @Test + void prepareUri_UriWithoutPort() { + FileServerData serverData = + FileServerData.builder().serverAddress(XNF_ADDRESS).userId(USERNAME).password(PASSWORD).build(); + String REMOTE_FILE = "any"; + + String retrievedUri = HttpUtils.prepareUri("http", serverData, REMOTE_FILE, 80); + assertTrue(retrievedUri.startsWith("http://" + XNF_ADDRESS + ":80")); + } + + @Test + void prepareUri_verifyUriWithTokenAndFragment() throws URISyntaxException { + String file = "/file"; + String expected = "http://" + XNF_ADDRESS + ":" + PORT + file + "?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&" + + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "#" + FRAGMENT; + assertEquals(expected, + HttpUtils.prepareUri("http", fileServerDataWithJWTTokenLongQueryAndFragment(), file, 443)); + } + + @Test + void prepareUri_verifyUriWithoutTokenAndWithoutFragment() throws URISyntaxException { + String file = "/file"; + String expected = "http://" + XNF_ADDRESS + ":" + PORT + file; + assertEquals(expected, HttpUtils.prepareUri("http", fileServerDataNoTokenNoFragment(), file, 443)); + } + + private List validTokenSingleQueryData() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + return new URIBuilder(query).getQueryParams(); + } + + private List validTokenDoubleQueryData() throws URISyntaxException { + StringBuilder doubleQuery = new StringBuilder(); + doubleQuery.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&"); + doubleQuery.append(ACCESS_TOKEN + "=" + JWT_PASSWORD); + return new URIBuilder(doubleQuery.toString()).getQueryParams(); + } + + private List validQueryNoToken() throws URISyntaxException { + String query = "?" + ANOTHER_TOKEN + "=" + JWT_PASSWORD; + return new URIBuilder(query).getQueryParams(); + } + + private List queryDataDoubleToken() throws URISyntaxException { + StringBuilder doubleToken = new StringBuilder(); + doubleToken.append("?" + ACCESS_TOKEN + "=" + JWT_PASSWORD + "&"); + doubleToken.append(ACCESS_TOKEN + "=" + JWT_PASSWORD + "&"); + doubleToken.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA); + return new URIBuilder(doubleToken.toString()).getQueryParams(); + } + + private FileServerData fileServerDataWithJWTToken() throws URISyntaxException { + String query = "?" + ACCESS_TOKEN + "=" + JWT_PASSWORD; + + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId("").password("").port(PORT) + .queryParameters(new URIBuilder(query).getQueryParams()).build(); + } + + private FileServerData fileServerDataWithJWTTokenLongQueryAndFragment() throws URISyntaxException { + StringBuilder query = new StringBuilder(); + query.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&"); + query.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA + "&"); + query.append(ACCESS_TOKEN + "=" + JWT_PASSWORD + "&"); + query.append(ANOTHER_TOKEN + "=" + ANOTHER_DATA); + + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId("").password("").port(PORT) + .queryParameters(new URIBuilder(query.toString()).getQueryParams()).uriRawFragment(FRAGMENT).build(); + } + + private FileServerData fileServerDataQueryWithoutToken() throws URISyntaxException { + StringBuilder query = new StringBuilder(); + query.append("?" + ANOTHER_TOKEN + "=" + ANOTHER_DATA); + + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId("").password("").port(PORT) + .queryParameters(new URIBuilder(query.toString()).getQueryParams()).build(); + } + + private FileServerData fileServerDataNoTokenNoFragment() throws URISyntaxException { + return FileServerData.builder().serverAddress(XNF_ADDRESS).userId("").password("").port(PORT) + .queryParameters(new URIBuilder("").getQueryParams()).uriRawFragment("").build(); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java new file mode 100644 index 0000000..6d437ae --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/tasks/FileCollectorTest.java @@ -0,0 +1,368 @@ +/*- + * ============LICENSE_START====================================================================== + * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved. + * Copyright (C) 2020-2022 Nokia. All rights reserved. + * =============================================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * ============LICENSE_END======================================================================== + */ + +package org.onap.dcaegen2.collectors.datafile.tasks; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig; +import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig; +import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException; +import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient; +import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient; +import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient; +import org.onap.dcaegen2.collectors.datafile.model.Counters; +import org.onap.dcaegen2.collectors.datafile.model.FileData; +import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation; +import org.onap.dcaegen2.collectors.datafile.model.FileReadyMessage; +import reactor.test.StepVerifier; + +public class FileCollectorTest { + + final static String DATAFILE_TMPDIR = "/tmp/onap_datafile/"; + private static final String PRODUCT_NAME = "NrRadio"; + private static final String VENDOR_NAME = "Ericsson"; + private static final int LAST_EPOCH_MICROSEC = 87457457; + private static final String SOURCE_NAME = "oteNB5309"; + private static final int START_EPOCH_MICROSEC = 874575764; + private static final String TIME_ZONE_OFFSET = "UTC+05:00"; + private static final String FTPES_SCHEME = "ftpes://"; + private static final String SFTP_SCHEME = "sftp://"; + private static final String HTTP_SCHEME = "http://"; + private static final String HTTPS_SCHEME = "https://"; + private static final String SERVER_ADDRESS = "192.168.0.101"; + private static final int PORT_22 = 22; + private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz"; + private static final Path LOCAL_FILE_LOCATION = Paths.get(DATAFILE_TMPDIR, SOURCE_NAME, PM_FILE_NAME); + private static final String REMOTE_FILE_LOCATION = "/ftp/rop/" + PM_FILE_NAME; + private static final String USER = "usr"; + private static final String PWD = "pwd"; + private static final String FTPES_LOCATION = + FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION; + + private static final String FTPES_LOCATION_NO_PORT = + FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION; + private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION; + private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION; + + private static final String HTTP_LOCATION = + HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION; + private static final String HTTP_LOCATION_NO_PORT = + HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION; + private static final String HTTPS_LOCATION = + HTTPS_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION; + private static final String HTTPS_LOCATION_NO_PORT = + HTTPS_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION; + + private static final String GZIP_COMPRESSION = "gzip"; + private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec"; + private static final String FILE_FORMAT_VERSION = "V10"; + private static final String CERTIFICATE_KEY_PASSWORD_PATH = "certificateKeyPassword"; + private static final String TRUSTED_CA_PATH = "trustedCAPath"; + private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword"; + private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES"; + private static final String FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec"; + private static final String CHANGE_TYPE = "FileReady"; + + private static AppConfig appConfigMock = mock(AppConfig.class); + private static CertificateConfig certificateConfigMock = mock(CertificateConfig.class); + + private FtpesClient ftpesClientMock = mock(FtpesClient.class); + + private SftpClient sftpClientMock = mock(SftpClient.class); + + private DfcHttpClient dfcHttpClientMock = mock(DfcHttpClient.class); + private DfcHttpsClient dfcHttpsClientMock = mock(DfcHttpsClient.class); + + private Counters counters; + + FileReadyMessage.Event event(String location) { + FileReadyMessage.MessageMetaData messageMetaData = FileReadyMessage.MessageMetaData.builder() // + .lastEpochMicrosec(LAST_EPOCH_MICROSEC) // + .sourceName(SOURCE_NAME) // + .startEpochMicrosec(START_EPOCH_MICROSEC) // + .timeZoneOffset(TIME_ZONE_OFFSET) // + .changeIdentifier(CHANGE_IDENTIFIER) // + .eventName("Noti_NrRadio-Ericsson_FileReady").build(); + + FileReadyMessage.FileInfo fileInfo = FileReadyMessage.FileInfo // + .builder() // + .fileFormatType(FILE_FORMAT_TYPE) // + .location(location) // + .fileFormatVersion(FILE_FORMAT_VERSION) // + .compression(GZIP_COMPRESSION) // + .build(); + + FileReadyMessage.ArrayOfNamedHashMap arrayOfNamedHashMap = FileReadyMessage.ArrayOfNamedHashMap // + .builder().name(PM_FILE_NAME) // + .hashMap(fileInfo).build(); + + List arrayOfNamedHashMapList = new ArrayList<>(); + arrayOfNamedHashMapList.add(arrayOfNamedHashMap); + + FileReadyMessage.NotificationFields notificationFields = FileReadyMessage.NotificationFields // + .builder().notificationFieldsVersion("notificationFieldsVersion") // + .changeType(CHANGE_TYPE).changeIdentifier(CHANGE_IDENTIFIER) // + .arrayOfNamedHashMap(arrayOfNamedHashMapList) // + .build(); + + return FileReadyMessage.Event.builder() // + .commonEventHeader(messageMetaData) // + .notificationFields(notificationFields).build(); + } + + private FileReadyMessage fileReadyMessage(String location) { + FileReadyMessage message = FileReadyMessage.builder() // + .event(event(location)) // + .build(); + return message; + } + + private FileData createFileData(String location) { + return FileData.createFileData(fileReadyMessage(location)).iterator().next(); + } + + private FilePublishInformation createExpectedFilePublishInformation(String location) { + return FilePublishInformation.builder() // + .productName(PRODUCT_NAME) // + .vendorName(VENDOR_NAME) // + .lastEpochMicrosec(LAST_EPOCH_MICROSEC) // + .sourceName(SOURCE_NAME) // + .startEpochMicrosec(START_EPOCH_MICROSEC) // + .timeZoneOffset(TIME_ZONE_OFFSET) // + .name(SOURCE_NAME + "/" + PM_FILE_NAME) // + .compression(GZIP_COMPRESSION) // + .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) // + .fileFormatVersion(FILE_FORMAT_VERSION) // + .changeIdentifier(CHANGE_IDENTIFIER) // + .build(); + } + + @BeforeAll + static void setUpConfiguration() { + when(appConfigMock.getCertificateConfiguration()).thenReturn(certificateConfigMock); + appConfigMock.collectedFilesPath = DATAFILE_TMPDIR; + certificateConfigMock.keyPasswordPath = CERTIFICATE_KEY_PASSWORD_PATH; + certificateConfigMock.trustedCa = TRUSTED_CA_PATH; + certificateConfigMock.trustedCaPasswordPath = TRUSTED_CA_PASSWORD_PATH; + } + + @BeforeEach + void setUpTest() { + counters = new Counters(); + } + + @Test + public void whenFtpesFile_returnCorrectResponse() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any()); + + FileData fileData = createFileData(FTPES_LOCATION_NO_PORT); + + FilePublishInformation expectedfilePublishInformation = + createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + verify(ftpesClientMock, times(1)).open(); + verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(ftpesClientMock, times(1)).close(); + verifyNoMoreInteractions(ftpesClientMock); + + assertEquals(1, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1"); + assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0"); + } + + @Test + public void whenSftpFile_returnCorrectResponse() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(sftpClientMock).when(collectorUndetTest).createSftpClient(any()); + + FileData fileData = createFileData(SFTP_LOCATION_NO_PORT); + FilePublishInformation expectedfilePublishInformation = + createExpectedFilePublishInformation(SFTP_LOCATION_NO_PORT); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + // The same again, but with port + fileData = createFileData(SFTP_LOCATION); + expectedfilePublishInformation = createExpectedFilePublishInformation(SFTP_LOCATION); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + verify(sftpClientMock, times(2)).open(); + verify(sftpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(sftpClientMock, times(2)).close(); + verifyNoMoreInteractions(sftpClientMock); + + assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 2"); + } + + @Test + public void whenHttpFile_returnCorrectResponse() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(dfcHttpClientMock).when(collectorUndetTest).createHttpClient(any()); + + FileData fileData = createFileData(HTTP_LOCATION_NO_PORT); + + FilePublishInformation expectedfilePublishInformation = + createExpectedFilePublishInformation(HTTP_LOCATION_NO_PORT); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + // The same again, but with port + fileData = createFileData(HTTP_LOCATION); + expectedfilePublishInformation = createExpectedFilePublishInformation(HTTP_LOCATION); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + verify(dfcHttpClientMock, times(2)).open(); + verify(dfcHttpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(dfcHttpClientMock, times(2)).close(); + verifyNoMoreInteractions(dfcHttpClientMock); + + assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1"); + assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0"); + } + + @Test + public void whenHttpsFile_returnCorrectResponse() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(dfcHttpsClientMock).when(collectorUndetTest).createHttpsClient(any()); + + FileData fileData = createFileData(HTTPS_LOCATION_NO_PORT); + + FilePublishInformation expectedfilePublishInformation = + createExpectedFilePublishInformation(HTTPS_LOCATION_NO_PORT); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + // The same again, but with port + fileData = createFileData(HTTPS_LOCATION); + expectedfilePublishInformation = createExpectedFilePublishInformation(HTTPS_LOCATION); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + verify(dfcHttpsClientMock, times(2)).open(); + verify(dfcHttpsClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + verify(dfcHttpsClientMock, times(2)).close(); + verifyNoMoreInteractions(dfcHttpsClientMock); + + assertEquals(2, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1"); + assertEquals(0, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 0"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0"); + } + + @Test + public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any()); + + FileData fileData = createFileData(FTPES_LOCATION); + doThrow(new DatafileTaskException("Unable to collect file.")).when(ftpesClientMock) + .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectErrorMessage("Retries exhausted: 3/3") // + .verify(); + + verify(ftpesClientMock, times(4)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + + assertEquals(0, counters.getNoOfCollectedFiles(), "collectedFiles should have been 0"); + assertEquals(4, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 4"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0"); + } + + @Test + public void whenFtpesFileAlwaysFail_failWithoutRetry() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any()); + + FileData fileData = createFileData(FTPES_LOCATION); + doThrow(new NonRetryableDatafileTaskException("Unable to collect file.")).when(ftpesClientMock) + .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectErrorMessage("Non retryable file transfer failure") // + .verify(); + + verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + + assertEquals(0, counters.getNoOfCollectedFiles(), "collectedFiles should have been 0"); + assertEquals(1, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 1"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0"); + } + + @Test + public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() throws Exception { + FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters)); + doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any()); + doThrow(new DatafileTaskException("Unable to collect file.")).doNothing().when(ftpesClientMock) + .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + + FilePublishInformation expectedfilePublishInformation = + createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT); + + FileData fileData = createFileData(FTPES_LOCATION_NO_PORT); + + StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0))) + .expectNext(expectedfilePublishInformation) // + .verifyComplete(); + + verify(ftpesClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION); + + assertEquals(1, counters.getNoOfCollectedFiles(), "collectedFiles should have been 1"); + assertEquals(1, counters.getNoOfFailedFtpAttempts(), "failedFtpAttempts should have been 1"); + assertEquals(0, counters.getNoOfFailedHttpAttempts(), "failedHttpAttempts should have been 0"); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/JsonMessage.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/JsonMessage.java new file mode 100644 index 0000000..7c2706d --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/JsonMessage.java @@ -0,0 +1,262 @@ +/*- + * ============LICENSE_START======================================================= + * 2018-2019 Nordix Foundation. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.utils; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Utility class to produce correctly formatted fileReady event Json messages. + * + * @author Henrik Andersson on 7/25/18 + * + */ +public class JsonMessage { + private String eventName; + private String changeIdentifier; + private String changeType; + private String notificationFieldsVersion; + private List arrayOfAdditionalFields; + + public List getAdditionalFields() { + return arrayOfAdditionalFields; + } + + @Override + public String toString() { + return "[" + getParsed() + "]"; + } + + /** + * Gets the message in parsed format. + * + * @return the massage in parsed format. + */ + public String getParsed() { + StringBuffer additionalFieldsString = new StringBuffer(); + if (arrayOfAdditionalFields.size() > 0) { + additionalFieldsString.append("\"arrayOfNamedHashMap\":["); + for (Iterator iterator = arrayOfAdditionalFields.iterator(); iterator.hasNext();) { + AdditionalField additionalField = iterator.next(); + additionalFieldsString.append(additionalField.toString()); + if (iterator.hasNext()) { + additionalFieldsString.append(","); + } + } + additionalFieldsString.append("]"); + } + return "{" // + + "\"event\":" // + + "{" // + + "\"commonEventHeader\":" // + + "{" // + + "\"domain\":\"notification\"," // + + "\"eventId\":\"<>-reg\"," // + + "\"eventName\":\"" + eventName + "\"," // + + "\"eventType\":\"fileReady\"," // + + "\"internalHeaderFields\":{}," // + + "\"lastEpochMicrosec\":1519837825682," // + + "\"nfNamingCode\":\"5GRAN\"," // + + "\"nfcNamingCode\":\"5DU\"," // + + "\"priority\":\"Normal\"," // + + "\"reportingEntityName\":\"5GRAN_DU\"," // + + "\"sequence\":0," // + + "\"sourceId\":\"<>\"," // + + "\"sourceName\":\"5GRAN_DU\"," // + + "\"timeZoneOffset\":\"UTC+05:00\"," // + + "\"startEpochMicrosec\":\"1519837825682\"," // + + "\"version\":3" // + + "}," // + + "\"notificationFields\":" // + + "{" // + + getAsStringIfParameterIsSet("changeIdentifier", changeIdentifier, + changeType != null || notificationFieldsVersion != null || arrayOfAdditionalFields.size() > 0) + + getAsStringIfParameterIsSet("changeType", changeType, + notificationFieldsVersion != null || arrayOfAdditionalFields.size() > 0) + + getAsStringIfParameterIsSet("notificationFieldsVersion", notificationFieldsVersion, + arrayOfAdditionalFields.size() > 0) + + additionalFieldsString.toString() // + + "}" // + + "}" // + + "}"; + } + + private JsonMessage(final JsonMessageBuilder builder) { + this.eventName = builder.eventName; + this.changeIdentifier = builder.changeIdentifier; + this.changeType = builder.changeType; + this.notificationFieldsVersion = builder.notificationFieldsVersion; + this.arrayOfAdditionalFields = builder.arrayOfAdditionalFields; + } + + public static class AdditionalField { + private String name; + private String location; + private String compression; + private String fileFormatType; + private String fileFormatVersion; + + @Override + public String toString() { + return "{" // + + getAsStringIfParameterIsSet("name", name, true) // + + "\"hashMap\":" // + + "{" + + getAsStringIfParameterIsSet("location", location, + compression != null || fileFormatType != null || fileFormatVersion != null) + + getAsStringIfParameterIsSet("compression", compression, + fileFormatType != null || fileFormatVersion != null) + + getAsStringIfParameterIsSet("fileFormatType", fileFormatType, fileFormatVersion != null) + + getAsStringIfParameterIsSet("fileFormatVersion", fileFormatVersion, false) // + + "}" // + + "}"; + } + + private AdditionalField(AdditionalFieldBuilder builder) { + this.name = builder.name; + this.location = builder.location; + this.compression = builder.compression; + this.fileFormatType = builder.fileFormatType; + this.fileFormatVersion = builder.fileFormatVersion; + } + + } + + public static class AdditionalFieldBuilder { + private String name; + private String location; + private String compression; + private String fileFormatType; + private String fileFormatVersion; + + public AdditionalFieldBuilder name(String name) { + this.name = name; + return this; + } + + public AdditionalFieldBuilder location(String location) { + this.location = location; + return this; + } + + public AdditionalFieldBuilder compression(String compression) { + this.compression = compression; + return this; + } + + public AdditionalFieldBuilder fileFormatType(String fileFormatType) { + this.fileFormatType = fileFormatType; + return this; + } + + public AdditionalFieldBuilder fileFormatVersion(String fileFormatVersion) { + this.fileFormatVersion = fileFormatVersion; + return this; + } + + public AdditionalField build() { + return new AdditionalField(this); + } + } + + public static class JsonMessageBuilder { + private String eventName; + private String changeIdentifier; + private String changeType; + private String notificationFieldsVersion; + private List arrayOfAdditionalFields = new ArrayList(); + + public JsonMessageBuilder eventName(String eventName) { + this.eventName = eventName; + return this; + } + + public JsonMessageBuilder changeIdentifier(String changeIdentifier) { + this.changeIdentifier = changeIdentifier; + return this; + } + + public JsonMessageBuilder changeType(String changeType) { + this.changeType = changeType; + return this; + } + + public JsonMessageBuilder notificationFieldsVersion(String notificationFieldsVersion) { + this.notificationFieldsVersion = notificationFieldsVersion; + return this; + } + + public JsonMessageBuilder addAdditionalField(AdditionalField additionalField) { + this.arrayOfAdditionalFields.add(additionalField); + return this; + } + + public JsonMessage build() { + return new JsonMessage(this); + } + } + + private static String getAsStringIfParameterIsSet(String parameterName, String parameterValue, + boolean withSeparator) { + String result = ""; + if (parameterValue != null) { + result = "\"" + parameterName + "\":\"" + parameterValue + "\""; + + if (withSeparator) { + result = result + ","; + } + } + return result; + } + + /** + * Can be used to produce a correct test Json message. Tip! Check the formatting with + * Json formatter + * + * @param args Not used + */ + public static void main(String[] args) { + AdditionalField additionalField = new JsonMessage.AdditionalFieldBuilder() // + .name("A20161224.1030-1045.bin.gz") // + .location("ftpes://192.168.0.101:22/ftp/rop/A20161224.1030-1045.bin.gz") // + .compression("gzip") // + .fileFormatType("org.3GPP.32.435#measCollec") // + .fileFormatVersion("V10") // + .build(); + AdditionalField secondAdditionalField = new JsonMessage.AdditionalFieldBuilder() // + .name("A20161224.1030-1045.bin.gz") // + .location("sftp://192.168.0.101:22/ftp/rop/A20161224.1030-1045.bin.gz") // + .compression("gzip") // + .fileFormatType("org.3GPP.32.435#measCollec") // + .fileFormatVersion("V10") // + .build(); + JsonMessage message = new JsonMessage.JsonMessageBuilder() // + .eventName("Noti_NrRadio-Ericsson_FileReady") // + .changeIdentifier("PM_MEAS_FILES") // + .changeType("FileReady") // + .notificationFieldsVersion("2.0") // + .addAdditionalField(additionalField) // + .addAdditionalField(secondAdditionalField) // + .build(); + System.out.println(message.toString()); + } +} diff --git a/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/LoggingUtils.java b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/LoggingUtils.java new file mode 100644 index 0000000..cfcb7bf --- /dev/null +++ b/datafilecollector/src/test/java/org/onap/dcaegen2/collectors/datafile/utils/LoggingUtils.java @@ -0,0 +1,56 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2019 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.dcaegen2.collectors.datafile.utils; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; + +import org.slf4j.LoggerFactory; + +public class LoggingUtils { + + /** + * Returns a ListAppender that contains all logging events. Call this method at the very beginning of the test + */ + public static ListAppender getLogListAppender(Class logClass) { + return getLogListAppender(logClass, false); + } + + /** + * Returns a ListAppender that contains all logging events. Call this method at the very beginning of the test + * + * @param logClass class whose appender is wanted. + * @param allLevels true if all log levels should be activated. + */ + public static ListAppender getLogListAppender(Class logClass, boolean allLevels) { + Logger logger = (Logger) LoggerFactory.getLogger(logClass); + if (allLevels) { + logger.setLevel(Level.ALL); + } + ListAppender listAppender = new ListAppender<>(); + listAppender.start(); + logger.addAppender(listAppender); + + return listAppender; + } +} diff --git a/datafilecollector/src/test/resources/cert.jks b/datafilecollector/src/test/resources/cert.jks new file mode 100755 index 0000000000000000000000000000000000000000..ff0e95ce35aeffeb13d108956ecdcb1b718e7312 GIT binary patch literal 4888 zcmb_fc|6oxAD$UwnXyKp1!K)N(~luE*S?f}YwU5SF__F4W+qE%F`|${*0*p)mMBYQ ziHbr|wku0W5+#zGC86{VbMO6do8I^Hem?IXGjpEjd(QLyo$oonbDmk9UY!PkKoI5! zy#5jP#^VSEYhNfz04@+kq53jIL047wHi1B#5P$()1q8U@DqLVFSQ)Yf4CVwu8PLgZ z$trvFq_D<)+ZTkKj@62ukyJ{6^X3`7%XYnl>55`xI#f^WCR~_Z#afn6_-mi)cHphc zcPKCAh<2XtIH~{D&c!gRnQ-L8En&a!u_5L?<_yn9Ojc%Vw_^ROwNl6N*+{RUyz07J zW78--S6jIo`d63iH6XGFXQf>Ux#v@a!*jY!5@7X`STB+WQ42s0*dyMV8^Ke%;il@>zst>m-f$oe~ zu9X9DfwQ$iO;(>SOUo7MW#w}HF1LIK9pc@M+?>63IQ{Oolv z{+{DnH_KjsEmK+j-PY9Q=(AB12?pVZaIMe54ZqA&%SISm?sjNpv$FMPRIn|#rvr?y}n*KnSRq=tBO8nZaK7%oLdJC4J zwec8rK?L~4D}6idQB#zEQDH=RAh+6R(!jaSB=m{*XC!!9FKo;4ccm6KE}j9qR9RG6 z4BEiq1*!|(o$oBPWI`AQXn9Qs{jY{mc>E~|sM{ZRy7}ZVJ0{*-x-s3J=0#aZ8GVv> z@!~hb8a;0k_+|3Kquu(RDv$VyJ%QokhFng+xa-tZ!s>K`1#Gn!=jG-(#i(LUttQPNIChgtYj!&@;cz-#v9TxtVHjTBv%N0iqX+o9(-hVMdRc` zQSL1KEt)!s&$`wKXf~xam1OT&UtB~!e=Iw;~ZDv%4WNHbHlnAB!8cS>pXP>0Ih zM}t}M)T=Z_apz}k0^RzJ(j8WL{O8KHiJbY(Ne(yrq|%acLpF;@ecFAtrHq`MUOv&3xj35|)$MQ7YZmq`IwJzG zcE}xE^$T1KrN2YmZppQ1D}j{OqZUHi@1b6x59`Nq-Omv-AC}j4S82a>Qmsk*o)A{7 zCucbD^4HMG+Wf+E9!uG(5A7)HrzNKgN)yllfnfRo1BL+@JhH!WK)?_P4D`7Gn|fxl zJPqLIf+s2Nu#Rf{0Ph-r^FeKG4DHke0KPTI%g1T! zXlQ7rwjB^#!yJ4S`M5XtWy>s%?OP$6wkU ze9*OHTulm)U}=l~c(m^`F?ISO`9Gd*<8uQHu*}ApF;9OE23QcpIyP@f;Q|4}?w}x7~1amK94hgjf%U5A7%#5wSw+M8{#?5zA zYoa;F$k4CL;^-Il`%nn!hsOcgWnrqZ+SelfIQvR&G6mf`^Jv^Po@Xy%hn~T=!7%rh z7o9b4N1cZRmFY=#PxPmD=0Cl@I5>DoeCt?}nUsBj&%9)q)r0FjX%n8PdpqCPynE(j z`t8bvoQ@9-ajpCO^QY!oJ}78fdhODT?Z;nxb4C4XZP%w2Rod~OrJ=MzI(Dpb%NGl@ ztzXgQgA!;jA-Ob})CS(2g&~h4gjx%8URmeXchnr$JyP{mMY>5VDaHA&t}%oI3ONc-r0El^Ka$qF{!Qk&uC}rdwnlR2tY-lTU$&u zB+5&qgF|MV9d5i-ZaEXb;}gK(bX`lk1I-(d=l>dg@N zyU9{QY{qN`Z(Dn>ka!a1s5j9k4EblJ7(3X~EK${;$JitgTkq|SqvL#tBxWWGiA2ED znFZeUlK?9T>OV!%n}6#5~vK3&6p)A z&xT~91^crb&5yZaYxNH#)BIQph|QSIU>;Tvsvm>@uNG^l)F#MgIw^8@U09#!`Ac+!2UVAb9Z~@*VX4%EOf8hSo zWglRt>qjAxncIXA;7bf3Aj!UDIz#M7?oZvH1lZ^b{n?dzl)2H@RYf;2v~`Lls>q&a zqX{DhtQ|}gY8XS{?^EC z-=>;*B* zQIO_E5cbkzRPO3bK!u_9p(TI9Sp(dz;pyqKyi3CW4GWj66bmhge%8H#Vvi|ekt;SC*%fSigNTVfc|n) z&@kp-%cB4Ta){-mm{-+59Mu1G)vehiHElp!O-oH%Qw^=PW|Pnx;Q!B${WnKc+8Ojb zv9hXq&{{UMLs=;~XgE;|mUBSHGV_gSe&}RDcKw&Y*e8QfqMmx+{nr{bjLVtFm`Dl>O#`?PI!&X`PFuXx^m+OEu`nAeZ}pHA1dsW^tU_jmCm zy@d=K4k<|^(;hb|w`gS*;qX&tCm6QYAsjj&fj0(@RIVFC$d8D`F_)xJGnJ25Y+Fi{F>dOPOhh}j7z@h7p#Qc0lZq1d6hcrmspM{A{vN@ z++aS~85%%}<#@ya(bHQUvFKULsZRUyhR}aCD5@QwnVdEUgx5NVteGYrfQ$J90fRRi zD{wR`hx|`n)#gpb!GM54f+Ey^A>vDYt3Jwf9g8kk_MtpHPY=?L2N$ly6m+$dcyi)< zJZ6OABoE+PL$~E&P8`CwX4hxr6+8)9-q%9EB_-J0D2FNItsJs&sD2eN=YTcX%eyjO zn^9};U@2{B%AeqLB>cGrQKX#lI3K~k4U+kyW604hjix#xZ*Q{Aqhi+!&JS^V%jwPY zZMX(-dyYEA zm+ty?sZ{?hx0oYcy*bYGHgi@TuJfEv46lu}OG`>L+bzxQ8S9_iU$;$? z=Svrjq;=Qdmot~&T+7Q~cW;=Pjn9WR1%uBCuf=4NfRH1M4`4)$4cR8|_5M>^;ODkg zqgq4k+fSK3aMi4OS_)4kh+qQ7?;H=wLlsR&)TmcqJ$K+(ioyr|dqLy9=))un-7l9D z=9;XlxfQ05m^qx2gx$CbaqOlVzc{w%d_y)+dHIvAg1omu9Y3kR$F6d5yizM|&sl5F ze(TOOThTuC<(C7+5nbC~VsqPL#pBKvapA>mmQn7n39XC3cMFlAIeGx%NtHPxG0Vu% m<(2pYW2?8wFiG5__bdIXM}0-%8t$-pzJSp0#Zn$xlm7dq<%1cK}(_;>Pp`+A{0 z{QW#a_bPr1c%e2B2o42Euw8(k3!%ydhJjTef?zNl1S7$grw0jlFZAlpW~)e8N0e$l zZmDk6emutI)JPSve~s*6`Wsfoy-5*ia1M7InX0&`Hu0LQyS?p{R;I4*fV;tom5iMf ziJJ~ijsk8Y%g59E7^tSxrSj2{`OAdzb%}d{FPAKnp8jJ24aS$KVaYcXpL37ays6UQIY<*?@(EGW>mhll1>1ybbj#ib( ziuYcpM`*o&LX_tg$|l2YvzT_K&6O6fcn#O*;*hw7EO`}clptY!Xf9mD+QbFE|<1l0eA_|a^)x&VRGuR&zb(6oWlR2lz zKbKAX(+lR3judyjP5J01iI6KKFEUgcVoQ@4=|n!)Xqn7Q`)WF07>b2hrOX>{SMpz| z{ZP&_Y@#B~kDNAmPn$!*^;RgWiOtpK_^`uJ^|p{^ zxWE#6wA1u(qzd^hZorgDfv#Ld<0Hey zLz4Oh;~i0=RIdu*25tv5)TmNUQTzc~<_k5sQLc`qJ%A1CccoC2+=D7feX}9bH?f5- z0oabM03?b^{}^iax%zrUTO{H1aO1(3eH+(`WO5b=@4$jvvj%T^|C(H0b7O)Jdl;Dd z*Thal6LdgPBSrQ3jW)F6d`}x4#Q?$u6mgN~>1ytK0_typS7;l%vlEY+V~q9OYT9&3 zZ&=m+n_z{->B@Usw)N9YSini0k!UK$%G07=ZT{|z?sAj2jk6rSyxT?7hx9`?vH{6# zFA07gJ@X?vQ*4uVxWs^Wto!~BARg#C+7P{796OA#@D_mByiKdeUex&bE!T94W%hO! zSHpPD*k~y!lv~?m#QE!*v+3SFPA9{kj(b#PFH22j#n(t8`>&^tPF+UdwDxNrz-(Lc z8TEK&0PnQ=%T(hdA6}-)O4+L5lY`JO`T2FHvglJ{as$$CYa{hozrNCrQ(AqRc|>iF zsg8DM*p*}@&m{M%vOxC6MviOQ#ao-LW0dtXbNY+nN0ZdP-$|hPdI2cY?l=_rp5KH)aJ~%%=zel2*DU zQfEg~;I$+#;dhM?_-!8gvD%d5lO${YzcQ*g>NV{5@Pbmyd>x~EIm~^Yf_Zn&p0snl zp#{}EpHJ{^QtBhgbO&*a2wefN>P9u@C&JukWMDACTv-J~fBR458g}&g|$eD2Y z+xXVGE>|eOcykawOdzaA<-EmyWgqtzXNP~3QU@d!4PD}IZWi9jzrqyMJT^$SCiZ9I zZ9@lON7IGpd0)qIfxk0Z?4LDv1_}nHTVEtE~o>HOBjMP(?9?` zzkmk`2L99Be`v%njRX+Cdian~Go1Z@1pQ5bj5ta|55Vk&E})44T*OhDz#eG*5C7le zNMMCO4edRACJy@NO0m(pXZQV^3wM$;GmgnK#L*PSK|r&&gW8VN~j zOend2%Uj8aBQC_o)G9reun^A>3u`(;p^i6yV*T)V9%8Rtj@ExdC@%MCw0=OJ@a?#k zTm89&NR7*wc8NosY*@3NX`y-At=+WLt*3VqTZVL?CxeWcgV)50uQk`Amb`Iz^XN^t=M?BuKjAR zk5(w!eKo0-#Op~g#Z5qbJjqDVQB}i!x9!J3p+3?n1!obL96eGN5YhHqGBQ&fYv)2P$PAK%^aVjF=Adj9DS)!f-E_ zK7Kuci#2JlS3MsqS1P>Pq!X3FNs-cZlK-S;`GQQ~wA%_ZvL()cMpX_!!!Ruem72?+-K2Bchqupv89qEd@nO*b~lRY$WvJ^VVcdQ xkikTZJ(WLdKKVmVcqQdT?S7w~W9LvN7-7LRtM40P=g_+X&(=RlW9?SD{skaHz3u=2 literal 0 HcmV?d00001 diff --git a/datafilecollector/src/test/resources/dfc.jks.pass b/datafilecollector/src/test/resources/dfc.jks.pass new file mode 100644 index 0000000..d97c5ea --- /dev/null +++ b/datafilecollector/src/test/resources/dfc.jks.pass @@ -0,0 +1 @@ +secret diff --git a/datafilecollector/src/test/resources/ftp.jks.pass b/datafilecollector/src/test/resources/ftp.jks.pass new file mode 100644 index 0000000..d97c5ea --- /dev/null +++ b/datafilecollector/src/test/resources/ftp.jks.pass @@ -0,0 +1 @@ +secret diff --git a/datafilecollector/src/test/resources/jks.pass b/datafilecollector/src/test/resources/jks.pass new file mode 100755 index 0000000..b2c3df4 --- /dev/null +++ b/datafilecollector/src/test/resources/jks.pass @@ -0,0 +1 @@ +hD:!w:CxF]lGvM6Mz9l^j[7U \ No newline at end of file diff --git a/datafilecollector/src/test/resources/keystore.p12 b/datafilecollector/src/test/resources/keystore.p12 new file mode 100644 index 0000000000000000000000000000000000000000..b847707c846ffdb1e13a61840d51728b88c82753 GIT binary patch literal 2857 zcmY+Ec{mh|7RF~7gQ1AAXD1Qv7-fwtW6d&Zh*8;hW8X5?$u0(262_82+4nMJ-_?jv zWUH|>LRnIlT=zcre&6%`@jmA~@B5zf=MRQs5~l&u!f=oUFr!qQe%!%1AU&`M2O)!S zkkK=H5Qd}k|3^gU2*T0Xp4oTLWC>>a-xV_iNK=FZ9l>y*1DGTj@_+m6IUfj`@L+`Q zf?+5o}BLS^j3gQOeG zzc>&2)9k$Gzlii|SUz>Ev@4n4U{LE}igQ}@qjIIR4NzzM*+lX%Vap>9l&J_pyqNli zgOUhfb)xp!bwF*dE99i{70NC=yKZS84J(uGs2TqO`!L zER6%7yTCJ&iTFJtWGCId`U9Y95vQu3Hev)DzU-U_pwZq1;Rc1x$sI%lhTcx zdWPg%*Q%v?IwrLX(zxk!7Z9D|W*+><;yDIC{f><2kfNjf5h{m_uX*s2?seaEEu>^Q zDsEIhIlXT4cDw9PSXKkIj<0U`*s_u^F;RnQ+Fl=>a67(k(Q?CSk3Fh-Qz@_N8(KG@ zaP-u@@sB23RO~v3&m*1|>DrjYn|;q!)oSkCRsV47?R5;x-NYvYj8AeUqGPW32{|M1TN(>Htav#Z@Wnkdp>=h65lzMX!cSh z0YSy~J|^5S96@}2u~Q}59*v)H;WvLzZ>P;mtd~kgcHK7*pMo0nyFzFoiij$q0@BwJ zJ71gP%{ed@HT^OQ8FnK3)RQA<{9L{MSW4kI+$!`pvyKo7-e=NB4w5O=BY{)1A+|Lg zRakkH=%n{L-zMi3#X^?_;Bpb11Lh ztdIJv#M!JGi{BfP@ABixqnss9hQ;Nep>m%ZE7vS2>0B1-dLOH&#W=%GNA0VKoFKN& zGM)q5`H*B6UP>~bRs7`p=)t-^%>s%erh1QO5ejpeQTD=ekNsInq6m*o&B$bVy1B4&1lj#LarL4`w*;cGzG9k+5{p@At! z#d(ssVS!gnXiU1gdJiL|&Q@U+$PL29WiR4;e8zgWxZ7(H`Tr8_*(2*ZU$`3lSo8r6O zQtGL*OxRMA=FGbv;~}eJm+4u9QBKt!n9Ean>s%T~zgXs>RXlljFAmU#20zL9 z`g;ALpw_7<)11sJx@NkHR1|U2MIlzzu?Skd^)`wXW`%U2^lD_!zkqnP{=fvrU2UiE zvv{Bvy37a#`8+EIgRTM_x%`sOv4z|59Iipu|yVPcb3;B z21tYI9#GWbh2;spuRbbz^MsYYa(Xtd4laH3U!>thyCZo|#b(sK>_)SVCo zult9srsL#=g0==^r?~2=7pR`U$%u@uzQ^OM8v%PzBm9SfzQY{6$(xh*B=@avmv$;D zXMSduRJcG1M7vCK>X(<-CQZ+?LEsS|FvF-gt3MRVxuw888NIt1SL&-0-ZK2Tdn|t{ zuryM^!5aZI!4-?a_J6lcg_RD^_9c4fY-4x)oPymwRP_DXmi2lEG9S8BVX;eii2IQv zI1CjO^sVA~fJY0jQ`Anq=$%^%ACN$M6B9wH>0LE;554d2EeHX@uf$a=IwwjGkCL_S zaiHv755q5~kHtow#P0P)yAPlUKeJmrzVbE7?|C)7x0p zCRk_9W*p|FvF6*vX_+qO-v$~LC8%&8aQAE1IKtyyc!)%MrvsVt3eJ854jA6GW&%LgiBN z(e!JAaE)Q1M!3(Yx6I6_`3q7{&;uyKuIR&G0&o t@U2-<*Qfdm71OoK4IrrQ_elK@K1!R4oOlZBY4$6gvh^_2)mqS>w zm`me7Y#624GscKh)6(P1XoJI(59 z7B+CllQi~YT=qpftgC^C&S}ZF<*2e6L7g3gri@s&qP$FK^kV1DV4zl&j@e+FG1Xc0 zY}TAwcO2ouvWZ}}-9gaO(7+DX9C35#k2Ufb=a7Hadq#Vv!YvBv9AACA8&_)zLu^ik zNWBOA&eP5i`s%WpZ$6#!O;#Awa(y#Q+~tXX9Na%GGHgOQcr^%OwNjIvDn*3p{h}8B z2m*D89T!sQvy%QH8P?>lpEz8UksC=q>^TyzBaBX_1})S- zvbk|Q&aJdr5|MbbJ#7u;D;a<7JS4uQU;4zVsn338SJag`ZhE-2A?vnfrgTfW6Jx)Q zsl$r6ukFsRdD`!b>U{TheqFfUxKAxP=6j*0i_GT!fq?4a(jn#>C#avpAel1NvhZfU zgq!n4+gTc*A|HihZA};;bH@Eo4$9-KESOlq#R_0R%j>4l32#KzrphfOP|{bivM zY=%Oz_t`{-9V5Rt+VX7mjR!@BL+4EZsh8w=PyFH4iG)1I*gGE&Bk{MI07c55mh#OM zW9_6*m6(H~4Ah7vww1W~;b1E(-0f>L;$6R|fAL982owwg7o7uGfbMz(b&)U(Y^y7S zv_PMhyNKJv?zR(%k#iqrjsbcbYeN_oz{LEw0kvU3gTZSsH~?J-0J$NcHU!vuLHIw$ zaJ?1lF$9CCdR^?wuzbfVsDzzQ4V?Pu5vD)YR9(;zGuF;4-*Do8{2*@Eb(6M0~=P+H~XHxsx~j1vVuHM zbb9ZuikSCS(hf64b?_BlO=CSp|LHA7vJvl%bCM=COqgG?AbS}WId!Qk8C&s6lpqL| zJlPQuk7w9dHZ;?Z_5~S$KX%pH4rU=0q4A%|kF{Yn4#oL5FT#zZA`kVOMG;O9p^%?O zKaa<3d&@KR7=yvZp@FPl+gwvq%=wwEB2!knXM32E=7pTdV~aipZU z%z%j=Xz{1LWedS&%RQ6B)Y`V)UuAURYxm3(1%m10qafV}1@qT^>ZY)4E^c7xAahP& z6`b?=WVC$Z>Z4LeFC%&U&c?gTsC%8FiBVkUwAoLoHMD>uN@r^Q-Byf2$BnT|F21My zc~-JXD(CyzM5e`-C2W+EE|Q6Ph9^>r$Wq+nk;v<)_n7Tb?*wx%UR}Uxh48l{*hh}7 T4(m&c$xCI6x|QekVp)F!c*kLN literal 0 HcmV?d00001 diff --git a/datafilecollector/src/test/resources/trust.pass b/datafilecollector/src/test/resources/trust.pass new file mode 100755 index 0000000..047a411 --- /dev/null +++ b/datafilecollector/src/test/resources/trust.pass @@ -0,0 +1 @@ +jeQ2l]iyB62D{WbSHL]dN*8R \ No newline at end of file -- 2.16.6