Storing EiJobs persistently

Created Ei jobs can then survive a restart of the container/POD

Change-Id: Ib3fede385b58f394f55cde068ba2ec5e1f4b0ebc
Signed-off-by: PatrikBuhr <patrik.buhr@est.tech>
Issue-ID: NONRTRIC-173

diff --git a/enrichment-coordinator-service/Dockerfile b/enrichment-coordinator-service/Dockerfile
index 51d45b5..744a237 100644 (file)
--- a/enrichment-coordinator-service/Dockerfile
+++ b/enrichment-coordinator-service/Dockerfile
@@ -24,6 +24,8 @@ ARG JAR
 WORKDIR /opt/app/enrichment-coordinator-service
 RUN mkdir -p /var/log/enrichment-coordinator-service
 RUN mkdir -p /opt/app/enrichment-coordinator-service/etc/cert/
+RUN mkdir -p /var/enrichment-coordinator-service
+RUN chmod -R 777 /var/enrichment-coordinator-service
 
 EXPOSE 8083 8434
 

diff --git a/enrichment-coordinator-service/config/application.yaml b/enrichment-coordinator-service/config/application.yaml
index e64db0c..850dc67 100644 (file)
--- a/enrichment-coordinator-service/config/application.yaml
+++ b/enrichment-coordinator-service/config/application.yaml
@@ -35,4 +35,5 @@ app:
     trust-store-used: false
     trust-store-password: policy_agent
     trust-store: /opt/app/enrichment-coordinator-service/etc/cert/truststore.jks
+  vardata-directory: /var/enrichment-coordinator-service
 

diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java
index f4cf9dc..c5d2bec 100644 (file)
--- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java
+++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java
@@ -22,11 +22,15 @@ package org.oransc.enrichment;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import java.lang.invoke.MethodHandles;
+
 import org.apache.catalina.connector.Connector;
 import org.oransc.enrichment.configuration.ApplicationConfig;
 import org.oransc.enrichment.repository.EiJobs;
 import org.oransc.enrichment.repository.EiProducers;
 import org.oransc.enrichment.repository.EiTypes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
@@ -40,6 +44,7 @@ class BeanFactory {
     private int httpPort = 0;
 
     private final ApplicationConfig applicationConfig = new ApplicationConfig();
+    private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
     @Bean
     public ObjectMapper mapper() {
@@ -57,7 +62,13 @@ class BeanFactory {
 
     @Bean
     public EiJobs eiJobs() {
-        return new EiJobs();
+        EiJobs jobs = new EiJobs(getApplicationConfig());
+        try {
+            jobs.restoreJobsFromDatabase();
+        } catch (Exception e) {
+            logger.error("Could not restore jobs from database: {}", e.getMessage());
+        }
+        return jobs;
     }
 
     @Bean

diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java
index 2d4087f..8937464 100644 (file)
--- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java
+++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java
@@ -36,6 +36,10 @@ public class ApplicationConfig {
     @Value("${app.filepath}")
     private String localConfigurationFilePath;
 
+    @Getter
+    @Value("${app.vardata-directory}")
+    private String vardataDirectory;
+
     @Value("${server.ssl.key-store-type}")
     private String sslKeyStoreType = "";
 

diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java
index 706c8dd..1532c53 100644 (file)
--- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java
+++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java
@@ -20,12 +20,29 @@
 
 package org.oransc.enrichment.repository;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapterFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.ServiceLoader;
 import java.util.Vector;
 
+import org.oransc.enrichment.configuration.ApplicationConfig;
 import org.oransc.enrichment.exceptions.ServiceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.FileSystemUtils;
 
 /**
  * Dynamic representation of all existing EI jobs.
@@ -35,11 +52,29 @@ public class EiJobs {
 
     private MultiMap<EiJob> jobsByType = new MultiMap<>();
     private MultiMap<EiJob> jobsByOwner = new MultiMap<>();
+    private final Gson gson;
+
+    private final ApplicationConfig config;
+    private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    public EiJobs(ApplicationConfig config) {
+        this.config = config;
+        GsonBuilder gsonBuilder = new GsonBuilder();
+        ServiceLoader.load(TypeAdapterFactory.class).forEach(gsonBuilder::registerTypeAdapterFactory);
+        this.gson = gsonBuilder.create();
+    }
+
+    public synchronized void restoreJobsFromDatabase() throws IOException {
+        File dbDir = new File(getDatabaseDirectory());
+        for (File file : dbDir.listFiles()) {
+            String json = Files.readString(file.toPath());
+            EiJob job = gson.fromJson(json, EiJob.class);
+            this.put(job, false);
+        }
+    }
 
     public synchronized void put(EiJob job) {
-        allEiJobs.put(job.id(), job);
-        jobsByType.put(job.typeId(), job.id(), job);
-        jobsByOwner.put(job.owner(), job.id(), job);
+        this.put(job, true);
     }
 
     public synchronized Collection<EiJob> getJobs() {
@@ -82,6 +117,13 @@ public class EiJobs {
         this.allEiJobs.remove(job.id());
         jobsByType.remove(job.typeId(), job.id());
         jobsByOwner.remove(job.owner(), job.id());
+
+        try {
+            Files.delete(getPath(job));
+        } catch (IOException e) {
+            logger.warn("Could not remove file: {}", e.getMessage());
+        }
+
     }
 
     public synchronized int size() {
@@ -92,6 +134,43 @@ public class EiJobs {
         this.allEiJobs.clear();
         this.jobsByType.clear();
         jobsByOwner.clear();
+        try {
+            FileSystemUtils.deleteRecursively(Path.of(getDatabaseDirectory()));
+        } catch (IOException e) {
+            logger.warn("Could not delete database : {}", e.getMessage());
+        }
+    }
+
+    private void put(EiJob job, boolean storePersistently) {
+        allEiJobs.put(job.id(), job);
+        jobsByType.put(job.typeId(), job.id(), job);
+        jobsByOwner.put(job.owner(), job.id(), job);
+        if (storePersistently) {
+            storeJobInFile(job);
+        }
+    }
+
+    private void storeJobInFile(EiJob job) {
+        try {
+            Files.createDirectories(Paths.get(getDatabaseDirectory()));
+            try (PrintStream out = new PrintStream(new FileOutputStream(getFile(job)))) {
+                out.print(gson.toJson(job));
+            }
+        } catch (Exception e) {
+            logger.warn("Could not save job: {} {}", job.id(), e.getMessage());
+        }
+    }
+
+    private File getFile(EiJob job) {
+        return getPath(job).toFile();
+    }
+
+    private Path getPath(EiJob job) {
+        return Path.of(getDatabaseDirectory(), job.id());
+    }
+
+    private String getDatabaseDirectory() {
+        return config.getVardataDirectory() + "/database";
     }
 
 }

diff --git a/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java b/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java
index e707fb7..30eaf68 100644 (file)
--- a/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java
+++ b/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java
@@ -88,7 +88,8 @@ import reactor.test.StepVerifier;
 @TestPropertySource(
     properties = { //
         "server.ssl.key-store=./config/keystore.jks", //
-        "app.webclient.trust-store=./config/truststore.jks"})
+        "app.webclient.trust-store=./config/truststore.jks", //
+        "app.vardata-directory=./target"})
 class ApplicationTest {
     private final String EI_TYPE_ID = "typeId";
     private final String EI_PRODUCER_ID = "producerId";
@@ -529,6 +530,33 @@ class ApplicationTest {
         assertThat(resp.getBody()).contains("hunky dory");
     }
 
+    @Test
+    void testEiJobDatabase() throws Exception {
+        putEiProducerWithOneType(EI_PRODUCER_ID, EI_TYPE_ID);
+        putEiJob(EI_TYPE_ID, "jobId1");
+        putEiJob(EI_TYPE_ID, "jobId2");
+
+        assertThat(this.eiJobs.size()).isEqualTo(2);
+
+        {
+            // Restore the jobs
+            EiJobs jobs = new EiJobs(this.applicationConfig);
+            jobs.restoreJobsFromDatabase();
+            assertThat(jobs.size()).isEqualTo(2);
+            jobs.remove("jobId1");
+            jobs.remove("jobId2");
+        }
+        {
+            // Restore the jobs, no jobs in database
+            EiJobs jobs = new EiJobs(this.applicationConfig);
+            jobs.restoreJobsFromDatabase();
+            assertThat(jobs.size()).isEqualTo(0);
+        }
+
+        this.eiJobs.remove("jobId1"); // removing a job when the db file is gone
+        assertThat(this.eiJobs.size()).isEqualTo(1);
+    }
+
     private void deleteEiProducer(String eiProducerId) {
         String url = ProducerConsts.API_ROOT + "/eiproducers/" + eiProducerId;
         restClient().deleteForEntity(url).block();