2 * ========================LICENSE_START=================================
5 * Copyright (C) 2023 Nordix Foundation
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ========================LICENSE_END===================================
21 package org.oran.pmproducer.datastore;
24 import java.nio.file.Path;
25 import java.util.concurrent.CompletableFuture;
27 import org.oran.pmproducer.configuration.ApplicationConfig;
28 import org.slf4j.Logger;
29 import org.slf4j.LoggerFactory;
31 import reactor.core.publisher.Flux;
32 import reactor.core.publisher.Mono;
33 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
34 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
35 import software.amazon.awssdk.core.BytesWrapper;
36 import software.amazon.awssdk.core.ResponseBytes;
37 import software.amazon.awssdk.core.async.AsyncRequestBody;
38 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
39 import software.amazon.awssdk.regions.Region;
40 import software.amazon.awssdk.services.s3.S3AsyncClient;
41 import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
42 import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
43 import software.amazon.awssdk.services.s3.model.CreateBucketResponse;
44 import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
45 import software.amazon.awssdk.services.s3.model.DeleteBucketResponse;
46 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
47 import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
48 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
49 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
50 import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
51 import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
52 import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
53 import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
54 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
55 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
56 import software.amazon.awssdk.services.s3.model.S3Object;
58 class S3ObjectStore implements DataStore {
59 private static final Logger logger = LoggerFactory.getLogger(S3ObjectStore.class);
60 private final ApplicationConfig applicationConfig;
62 private static S3AsyncClient s3AsynchClient;
64 public S3ObjectStore(ApplicationConfig applicationConfig) {
65 this.applicationConfig = applicationConfig;
67 getS3AsynchClient(applicationConfig);
70 @SuppressWarnings({"java:S3010", "java:S2209"})
71 public S3ObjectStore(ApplicationConfig applicationConfig, S3AsyncClient s3AsynchClient) {
72 this.applicationConfig = applicationConfig;
73 this.s3AsynchClient = s3AsynchClient;
76 private static synchronized S3AsyncClient getS3AsynchClient(ApplicationConfig applicationConfig) {
77 if (applicationConfig.isS3Enabled() && s3AsynchClient == null) {
78 s3AsynchClient = getS3AsyncClientBuilder(applicationConfig).build();
80 return s3AsynchClient;
83 private static S3AsyncClientBuilder getS3AsyncClientBuilder(ApplicationConfig applicationConfig) {
84 URI uri = URI.create(applicationConfig.getS3EndpointOverride());
85 return S3AsyncClient.builder() //
86 .region(Region.US_EAST_1) //
87 .endpointOverride(uri) //
88 .credentialsProvider(StaticCredentialsProvider.create( //
89 AwsBasicCredentials.create(applicationConfig.getS3AccessKeyId(), //
90 applicationConfig.getS3SecretAccessKey())));
94 public Flux<String> listObjects(Bucket bucket, String prefix) {
95 return listObjectsInBucket(bucket(bucket), prefix).map(S3Object::key);
99 public Mono<Boolean> createLock(String name) {
100 return getHeadObject(bucket(Bucket.LOCKS), name).flatMap(head -> createLock(name, head)) //
101 .onErrorResume(t -> createLock(name, null));
104 private Mono<Boolean> createLock(String name, HeadObjectResponse head) {
107 return this.putObject(Bucket.LOCKS, name, "") //
108 .flatMap(resp -> Mono.just(true)) //
109 .doOnError(t -> logger.warn("Failed to create lock {}, reason: {}", name, t.getMessage())) //
110 .onErrorResume(t -> Mono.just(false));
112 return Mono.just(false);
117 public Mono<Boolean> deleteLock(String name) {
118 return deleteObject(Bucket.LOCKS, name);
122 public Mono<Boolean> deleteObject(Bucket bucket, String name) {
124 DeleteObjectRequest request = DeleteObjectRequest.builder() //
125 .bucket(bucket(bucket)) //
129 CompletableFuture<DeleteObjectResponse> future = s3AsynchClient.deleteObject(request);
131 return Mono.fromFuture(future).map(resp -> true);
135 public Mono<byte[]> readObject(Bucket bucket, String fileName) {
136 return getDataFromS3Object(bucket(bucket), fileName);
139 public Mono<String> putObject(Bucket bucket, String fileName, String bodyString) {
140 PutObjectRequest request = PutObjectRequest.builder() //
141 .bucket(bucket(bucket)) //
145 AsyncRequestBody body = AsyncRequestBody.fromString(bodyString);
147 CompletableFuture<PutObjectResponse> future = s3AsynchClient.putObject(request, body);
149 return Mono.fromFuture(future) //
150 .map(putObjectResponse -> fileName) //
151 .doOnError(t -> logger.error("Failed to store file in S3 {}", t.getMessage()));
155 public Mono<String> copyFileTo(Path fromFile, String toFile) {
156 return copyFileToS3Bucket(bucket(Bucket.FILES), fromFile, toFile);
160 public Mono<String> create(Bucket bucket) {
161 return createS3Bucket(bucket(bucket));
164 private Mono<String> createS3Bucket(String s3Bucket) {
166 CreateBucketRequest request = CreateBucketRequest.builder() //
170 CompletableFuture<CreateBucketResponse> future = s3AsynchClient.createBucket(request);
172 return Mono.fromFuture(future) //
173 .map(f -> s3Bucket) //
174 .doOnError(t -> logger.trace("Could not create S3 bucket: {}", t.getMessage()))
175 .onErrorResume(t -> Mono.just(s3Bucket));
179 public Mono<String> deleteBucket(Bucket bucket) {
180 return listObjects(bucket, "") //
181 .flatMap(key -> deleteObject(bucket, key)) //
183 .flatMap(list -> deleteBucketFromS3Storage(bucket)) //
185 .doOnError(t -> logger.warn("Could not delete bucket: {}, reason: {}", bucket(bucket), t.getMessage()))
186 .onErrorResume(t -> Mono.just("NOK"));
189 private Mono<DeleteBucketResponse> deleteBucketFromS3Storage(Bucket bucket) {
190 DeleteBucketRequest request = DeleteBucketRequest.builder() //
191 .bucket(bucket(bucket)) //
194 CompletableFuture<DeleteBucketResponse> future = s3AsynchClient.deleteBucket(request);
196 return Mono.fromFuture(future);
199 private String bucket(Bucket bucket) {
200 return bucket == Bucket.FILES ? applicationConfig.getS3Bucket() : applicationConfig.getS3LocksBucket();
203 private Mono<String> copyFileToS3Bucket(String s3Bucket, Path fileName, String s3Key) {
205 PutObjectRequest request = PutObjectRequest.builder() //
210 AsyncRequestBody body = AsyncRequestBody.fromFile(fileName);
212 CompletableFuture<PutObjectResponse> future = s3AsynchClient.putObject(request, body);
214 return Mono.fromFuture(future) //
216 .doOnError(t -> logger.error("Failed to store file in S3 {}", t.getMessage()));
220 private Mono<HeadObjectResponse> getHeadObject(String bucket, String key) {
221 HeadObjectRequest request = HeadObjectRequest.builder().bucket(bucket).key(key).build();
223 CompletableFuture<HeadObjectResponse> future = s3AsynchClient.headObject(request);
224 return Mono.fromFuture(future);
227 private Mono<ListObjectsResponse> listObjectsRequest(String bucket, String prefix,
228 ListObjectsResponse prevResponse) {
229 ListObjectsRequest.Builder builder = ListObjectsRequest.builder() //
234 if (prevResponse != null) {
235 if (Boolean.TRUE.equals(prevResponse.isTruncated())) {
236 builder.marker(prevResponse.nextMarker());
242 ListObjectsRequest listObjectsRequest = builder.build();
243 CompletableFuture<ListObjectsResponse> future = s3AsynchClient.listObjects(listObjectsRequest);
244 return Mono.fromFuture(future);
247 private Flux<S3Object> listObjectsInBucket(String bucket, String prefix) {
249 return listObjectsRequest(bucket, prefix, null) //
250 .expand(response -> listObjectsRequest(bucket, prefix, response)) //
251 .map(ListObjectsResponse::contents) //
252 .doOnNext(f -> logger.debug("Found objects in {}: {}", bucket, f.size())) //
253 .doOnError(t -> logger.warn("Error fromlist objects: {}", t.getMessage())) //
254 .flatMap(Flux::fromIterable) //
255 .doOnNext(obj -> logger.debug("Found object: {}", obj.key()));
258 private Mono<byte[]> getDataFromS3Object(String bucket, String key) {
260 GetObjectRequest request = GetObjectRequest.builder() //
265 CompletableFuture<ResponseBytes<GetObjectResponse>> future =
266 s3AsynchClient.getObject(request, AsyncResponseTransformer.toBytes());
268 return Mono.fromFuture(future) //
269 .map(BytesWrapper::asByteArray) //
270 .doOnError(t -> logger.error("Failed to get file from S3, key:{}, bucket: {}, {}", key, bucket,
272 .doOnNext(n -> logger.debug("Read file from S3: {} {}", bucket, key)) //
273 .onErrorResume(t -> Mono.empty());