From: PatrikBuhr Date: Thu, 29 Oct 2020 10:58:43 +0000 (+0100) Subject: Storing EiJobs persistently X-Git-Tag: 2.1.0~20^2 X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=768cb9522bcb458171eddd5cc6213fb8e8797be3;p=nonrtric.git Storing EiJobs persistently Created Ei jobs can then survive a restart of the container/POD Change-Id: Ib3fede385b58f394f55cde068ba2ec5e1f4b0ebc Signed-off-by: PatrikBuhr Issue-ID: NONRTRIC-173 --- diff --git a/enrichment-coordinator-service/Dockerfile b/enrichment-coordinator-service/Dockerfile index 51d45b5d..744a237a 100644 --- a/enrichment-coordinator-service/Dockerfile +++ b/enrichment-coordinator-service/Dockerfile @@ -24,6 +24,8 @@ ARG JAR WORKDIR /opt/app/enrichment-coordinator-service RUN mkdir -p /var/log/enrichment-coordinator-service RUN mkdir -p /opt/app/enrichment-coordinator-service/etc/cert/ +RUN mkdir -p /var/enrichment-coordinator-service +RUN chmod -R 777 /var/enrichment-coordinator-service EXPOSE 8083 8434 diff --git a/enrichment-coordinator-service/config/application.yaml b/enrichment-coordinator-service/config/application.yaml index e64db0c9..850dc67f 100644 --- a/enrichment-coordinator-service/config/application.yaml +++ b/enrichment-coordinator-service/config/application.yaml @@ -35,4 +35,5 @@ app: trust-store-used: false trust-store-password: policy_agent trust-store: /opt/app/enrichment-coordinator-service/etc/cert/truststore.jks + vardata-directory: /var/enrichment-coordinator-service diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java index f4cf9dcc..c5d2bec7 100644 --- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java +++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/BeanFactory.java @@ -22,11 +22,15 @@ package org.oransc.enrichment; import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.invoke.MethodHandles; + import org.apache.catalina.connector.Connector; import org.oransc.enrichment.configuration.ApplicationConfig; import org.oransc.enrichment.repository.EiJobs; import org.oransc.enrichment.repository.EiProducers; import org.oransc.enrichment.repository.EiTypes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; import org.springframework.boot.web.servlet.server.ServletWebServerFactory; @@ -40,6 +44,7 @@ class BeanFactory { private int httpPort = 0; private final ApplicationConfig applicationConfig = new ApplicationConfig(); + private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Bean public ObjectMapper mapper() { @@ -57,7 +62,13 @@ class BeanFactory { @Bean public EiJobs eiJobs() { - return new EiJobs(); + EiJobs jobs = new EiJobs(getApplicationConfig()); + try { + jobs.restoreJobsFromDatabase(); + } catch (Exception e) { + logger.error("Could not restore jobs from database: {}", e.getMessage()); + } + return jobs; } @Bean diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java index 2d4087fa..89374649 100644 --- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java +++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/configuration/ApplicationConfig.java @@ -36,6 +36,10 @@ public class ApplicationConfig { @Value("${app.filepath}") private String localConfigurationFilePath; + @Getter + @Value("${app.vardata-directory}") + private String vardataDirectory; + @Value("${server.ssl.key-store-type}") private String sslKeyStoreType = ""; diff --git a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java index 706c8ddd..1532c535 100644 --- a/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java +++ b/enrichment-coordinator-service/src/main/java/org/oransc/enrichment/repository/EiJobs.java @@ -20,12 +20,29 @@ package org.oransc.enrichment.repository; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.TypeAdapterFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.ServiceLoader; import java.util.Vector; +import org.oransc.enrichment.configuration.ApplicationConfig; import org.oransc.enrichment.exceptions.ServiceException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.FileSystemUtils; /** * Dynamic representation of all existing EI jobs. @@ -35,11 +52,29 @@ public class EiJobs { private MultiMap jobsByType = new MultiMap<>(); private MultiMap jobsByOwner = new MultiMap<>(); + private final Gson gson; + + private final ApplicationConfig config; + private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public EiJobs(ApplicationConfig config) { + this.config = config; + GsonBuilder gsonBuilder = new GsonBuilder(); + ServiceLoader.load(TypeAdapterFactory.class).forEach(gsonBuilder::registerTypeAdapterFactory); + this.gson = gsonBuilder.create(); + } + + public synchronized void restoreJobsFromDatabase() throws IOException { + File dbDir = new File(getDatabaseDirectory()); + for (File file : dbDir.listFiles()) { + String json = Files.readString(file.toPath()); + EiJob job = gson.fromJson(json, EiJob.class); + this.put(job, false); + } + } public synchronized void put(EiJob job) { - allEiJobs.put(job.id(), job); - jobsByType.put(job.typeId(), job.id(), job); - jobsByOwner.put(job.owner(), job.id(), job); + this.put(job, true); } public synchronized Collection getJobs() { @@ -82,6 +117,13 @@ public class EiJobs { this.allEiJobs.remove(job.id()); jobsByType.remove(job.typeId(), job.id()); jobsByOwner.remove(job.owner(), job.id()); + + try { + Files.delete(getPath(job)); + } catch (IOException e) { + logger.warn("Could not remove file: {}", e.getMessage()); + } + } public synchronized int size() { @@ -92,6 +134,43 @@ public class EiJobs { this.allEiJobs.clear(); this.jobsByType.clear(); jobsByOwner.clear(); + try { + FileSystemUtils.deleteRecursively(Path.of(getDatabaseDirectory())); + } catch (IOException e) { + logger.warn("Could not delete database : {}", e.getMessage()); + } + } + + private void put(EiJob job, boolean storePersistently) { + allEiJobs.put(job.id(), job); + jobsByType.put(job.typeId(), job.id(), job); + jobsByOwner.put(job.owner(), job.id(), job); + if (storePersistently) { + storeJobInFile(job); + } + } + + private void storeJobInFile(EiJob job) { + try { + Files.createDirectories(Paths.get(getDatabaseDirectory())); + try (PrintStream out = new PrintStream(new FileOutputStream(getFile(job)))) { + out.print(gson.toJson(job)); + } + } catch (Exception e) { + logger.warn("Could not save job: {} {}", job.id(), e.getMessage()); + } + } + + private File getFile(EiJob job) { + return getPath(job).toFile(); + } + + private Path getPath(EiJob job) { + return Path.of(getDatabaseDirectory(), job.id()); + } + + private String getDatabaseDirectory() { + return config.getVardataDirectory() + "/database"; } } diff --git a/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java b/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java index e707fb78..30eaf68f 100644 --- a/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java +++ b/enrichment-coordinator-service/src/test/java/org/oransc/enrichment/ApplicationTest.java @@ -88,7 +88,8 @@ import reactor.test.StepVerifier; @TestPropertySource( properties = { // "server.ssl.key-store=./config/keystore.jks", // - "app.webclient.trust-store=./config/truststore.jks"}) + "app.webclient.trust-store=./config/truststore.jks", // + "app.vardata-directory=./target"}) class ApplicationTest { private final String EI_TYPE_ID = "typeId"; private final String EI_PRODUCER_ID = "producerId"; @@ -529,6 +530,33 @@ class ApplicationTest { assertThat(resp.getBody()).contains("hunky dory"); } + @Test + void testEiJobDatabase() throws Exception { + putEiProducerWithOneType(EI_PRODUCER_ID, EI_TYPE_ID); + putEiJob(EI_TYPE_ID, "jobId1"); + putEiJob(EI_TYPE_ID, "jobId2"); + + assertThat(this.eiJobs.size()).isEqualTo(2); + + { + // Restore the jobs + EiJobs jobs = new EiJobs(this.applicationConfig); + jobs.restoreJobsFromDatabase(); + assertThat(jobs.size()).isEqualTo(2); + jobs.remove("jobId1"); + jobs.remove("jobId2"); + } + { + // Restore the jobs, no jobs in database + EiJobs jobs = new EiJobs(this.applicationConfig); + jobs.restoreJobsFromDatabase(); + assertThat(jobs.size()).isEqualTo(0); + } + + this.eiJobs.remove("jobId1"); // removing a job when the db file is gone + assertThat(this.eiJobs.size()).isEqualTo(1); + } + private void deleteEiProducer(String eiProducerId) { String url = ProducerConsts.API_ROOT + "/eiproducers/" + eiProducerId; restClient().deleteForEntity(url).block();