2 * ========================LICENSE_START=================================
5 * Copyright (C) 2023 Nordix Foundation
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ========================LICENSE_END===================================
21 package org.oran.pmproducer.datastore;
24 import java.nio.file.Path;
25 import java.util.concurrent.CompletableFuture;
27 import org.oran.pmproducer.configuration.ApplicationConfig;
28 import org.slf4j.Logger;
29 import org.slf4j.LoggerFactory;
31 import reactor.core.publisher.Flux;
32 import reactor.core.publisher.Mono;
33 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
34 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
35 import software.amazon.awssdk.core.BytesWrapper;
36 import software.amazon.awssdk.core.ResponseBytes;
37 import software.amazon.awssdk.core.async.AsyncRequestBody;
38 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
39 import software.amazon.awssdk.regions.Region;
40 import software.amazon.awssdk.services.s3.S3AsyncClient;
41 import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
42 import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
43 import software.amazon.awssdk.services.s3.model.CreateBucketResponse;
44 import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
45 import software.amazon.awssdk.services.s3.model.DeleteBucketResponse;
46 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
47 import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
48 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
49 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
50 import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
51 import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
52 import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
53 import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
54 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
55 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
56 import software.amazon.awssdk.services.s3.model.S3Object;
58 class S3ObjectStore implements DataStore {
59 private static final Logger logger = LoggerFactory.getLogger(S3ObjectStore.class);
60 private final ApplicationConfig applicationConfig;
62 private static S3AsyncClient s3AsynchClient;
64 public S3ObjectStore(ApplicationConfig applicationConfig) {
65 this.applicationConfig = applicationConfig;
67 getS3AsynchClient(applicationConfig);
70 private static synchronized S3AsyncClient getS3AsynchClient(ApplicationConfig applicationConfig) {
71 if (applicationConfig.isS3Enabled() && s3AsynchClient == null) {
72 s3AsynchClient = getS3AsyncClientBuilder(applicationConfig).build();
74 return s3AsynchClient;
77 private static S3AsyncClientBuilder getS3AsyncClientBuilder(ApplicationConfig applicationConfig) {
78 URI uri = URI.create(applicationConfig.getS3EndpointOverride());
79 return S3AsyncClient.builder() //
80 .region(Region.US_EAST_1) //
81 .endpointOverride(uri) //
82 .credentialsProvider(StaticCredentialsProvider.create( //
83 AwsBasicCredentials.create(applicationConfig.getS3AccessKeyId(), //
84 applicationConfig.getS3SecretAccessKey())));
88 public Flux<String> listObjects(Bucket bucket, String prefix) {
89 return listObjectsInBucket(bucket(bucket), prefix).map(S3Object::key);
93 public Mono<Boolean> createLock(String name) {
94 return getHeadObject(bucket(Bucket.LOCKS), name).flatMap(head -> createLock(name, head)) //
95 .onErrorResume(t -> createLock(name, null));
98 private Mono<Boolean> createLock(String name, HeadObjectResponse head) {
101 return this.putObject(Bucket.LOCKS, name, "") //
102 .flatMap(resp -> Mono.just(true)) //
103 .doOnError(t -> logger.warn("Failed to create lock {}, reason: {}", name, t.getMessage())) //
104 .onErrorResume(t -> Mono.just(false));
106 return Mono.just(false);
111 public Mono<Boolean> deleteLock(String name) {
112 return deleteObject(Bucket.LOCKS, name);
116 public Mono<Boolean> deleteObject(Bucket bucket, String name) {
118 DeleteObjectRequest request = DeleteObjectRequest.builder() //
119 .bucket(bucket(bucket)) //
123 CompletableFuture<DeleteObjectResponse> future = s3AsynchClient.deleteObject(request);
125 return Mono.fromFuture(future).map(resp -> true);
129 public Mono<byte[]> readObject(Bucket bucket, String fileName) {
130 return getDataFromS3Object(bucket(bucket), fileName);
133 public Mono<String> putObject(Bucket bucket, String fileName, String bodyString) {
134 PutObjectRequest request = PutObjectRequest.builder() //
135 .bucket(bucket(bucket)) //
139 AsyncRequestBody body = AsyncRequestBody.fromString(bodyString);
141 CompletableFuture<PutObjectResponse> future = s3AsynchClient.putObject(request, body);
143 return Mono.fromFuture(future) //
144 .map(putObjectResponse -> fileName) //
145 .doOnError(t -> logger.error("Failed to store file in S3 {}", t.getMessage()));
149 public Mono<String> copyFileTo(Path fromFile, String toFile) {
150 return copyFileToS3Bucket(bucket(Bucket.FILES), fromFile, toFile);
154 public Mono<String> create(Bucket bucket) {
155 return createS3Bucket(bucket(bucket));
158 private Mono<String> createS3Bucket(String s3Bucket) {
160 CreateBucketRequest request = CreateBucketRequest.builder() //
164 CompletableFuture<CreateBucketResponse> future = s3AsynchClient.createBucket(request);
166 return Mono.fromFuture(future) //
167 .map(f -> s3Bucket) //
168 .doOnError(t -> logger.trace("Could not create S3 bucket: {}", t.getMessage()))
169 .onErrorResume(t -> Mono.just(s3Bucket));
173 public Mono<String> deleteBucket(Bucket bucket) {
174 return listObjects(bucket, "") //
175 .flatMap(key -> deleteObject(bucket, key)) //
177 .flatMap(list -> deleteBucketFromS3Storage(bucket)) //
179 .doOnError(t -> logger.warn("Could not delete bucket: {}, reason: {}", bucket(bucket), t.getMessage()))
180 .onErrorResume(t -> Mono.just("NOK"));
183 private Mono<DeleteBucketResponse> deleteBucketFromS3Storage(Bucket bucket) {
184 DeleteBucketRequest request = DeleteBucketRequest.builder() //
185 .bucket(bucket(bucket)) //
188 CompletableFuture<DeleteBucketResponse> future = s3AsynchClient.deleteBucket(request);
190 return Mono.fromFuture(future);
193 private String bucket(Bucket bucket) {
194 return bucket == Bucket.FILES ? applicationConfig.getS3Bucket() : applicationConfig.getS3LocksBucket();
197 private Mono<String> copyFileToS3Bucket(String s3Bucket, Path fileName, String s3Key) {
199 PutObjectRequest request = PutObjectRequest.builder() //
204 AsyncRequestBody body = AsyncRequestBody.fromFile(fileName);
206 CompletableFuture<PutObjectResponse> future = s3AsynchClient.putObject(request, body);
208 return Mono.fromFuture(future) //
210 .doOnError(t -> logger.error("Failed to store file in S3 {}", t.getMessage()));
214 private Mono<HeadObjectResponse> getHeadObject(String bucket, String key) {
215 HeadObjectRequest request = HeadObjectRequest.builder().bucket(bucket).key(key).build();
217 CompletableFuture<HeadObjectResponse> future = s3AsynchClient.headObject(request);
218 return Mono.fromFuture(future);
221 private Mono<ListObjectsResponse> listObjectsRequest(String bucket, String prefix,
222 ListObjectsResponse prevResponse) {
223 ListObjectsRequest.Builder builder = ListObjectsRequest.builder() //
228 if (prevResponse != null) {
229 if (Boolean.TRUE.equals(prevResponse.isTruncated())) {
230 builder.marker(prevResponse.nextMarker());
236 ListObjectsRequest listObjectsRequest = builder.build();
237 CompletableFuture<ListObjectsResponse> future = s3AsynchClient.listObjects(listObjectsRequest);
238 return Mono.fromFuture(future);
241 private Flux<S3Object> listObjectsInBucket(String bucket, String prefix) {
243 return listObjectsRequest(bucket, prefix, null) //
244 .expand(response -> listObjectsRequest(bucket, prefix, response)) //
245 .map(ListObjectsResponse::contents) //
246 .doOnNext(f -> logger.debug("Found objects in {}: {}", bucket, f.size())) //
247 .doOnError(t -> logger.warn("Error fromlist objects: {}", t.getMessage())) //
248 .flatMap(Flux::fromIterable) //
249 .doOnNext(obj -> logger.debug("Found object: {}", obj.key()));
252 private Mono<byte[]> getDataFromS3Object(String bucket, String key) {
254 GetObjectRequest request = GetObjectRequest.builder() //
259 CompletableFuture<ResponseBytes<GetObjectResponse>> future =
260 s3AsynchClient.getObject(request, AsyncResponseTransformer.toBytes());
262 return Mono.fromFuture(future) //
263 .map(BytesWrapper::asByteArray) //
264 .doOnError(t -> logger.error("Failed to get file from S3, key:{}, bucket: {}, {}", key, bucket,
266 .doOnNext(n -> logger.debug("Read file from S3: {} {}", bucket, key)) //
267 .onErrorResume(t -> Mono.empty());