From e0b56b40f659bd132f5ba7eb56576158e685a488 Mon Sep 17 00:00:00 2001 From: PatrikBuhr Date: Mon, 1 Aug 2022 13:24:48 +0200 Subject: [PATCH] NONRTRIC - dmaap adapter characteristic improvement Added support for PM filtering of the MO class of the measured object. Changed the ICS producer ID so that each instance will have a unique ID (using the callback URL). Changed the Kafka group ID so that each type will have an own group ID Setting kafka client ID (using the callback URL) Signed-off-by: PatrikBuhr Issue-ID: NONRTRIC-773 Change-Id: I1031802469beb146039ed089e4c80f0ca83d4dd9 --- docs/overview.rst | 6 +++-- .../dmaapadapter/controllers/ErrorResponse.java | 1 + .../controllers/ProducerCallbacksController.java | 2 +- .../java/org/oran/dmaapadapter/repository/Job.java | 2 +- .../repository/filters/JsonPathFilter.java | 2 +- .../repository/filters/PmReportFilter.java | 31 ++++++++++++++++++++-- .../dmaapadapter/tasks/DmaapTopicListener.java | 2 +- .../dmaapadapter/tasks/KafkaTopicListener.java | 3 ++- .../tasks/ProducerRegstrationTask.java | 15 ++++++----- src/main/resources/typeSchemaPmData.json | 8 ++++++ .../oran/dmaapadapter/IcsSimulatorController.java | 2 +- .../org/oran/dmaapadapter/IntegrationWithIcs.java | 2 +- .../oran/dmaapadapter/IntegrationWithKafka.java | 6 ++--- .../repository/filters/PmReportFilterTest.java | 17 +++++++++++- 14 files changed, 77 insertions(+), 22 deletions(-) diff --git a/docs/overview.rst b/docs/overview.rst index e0d6008..1174e44 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -186,6 +186,8 @@ The filterType parameter is extended to allow value "pmdata" which can be used f For instance a value like "NRCellCU" will match "ManagedElement=seliitdus00487,GNBCUCPFunction=1,NRCellCU=32". * measTypes selects the meas types to get * measuredEntityDns partial match of meas entity DNs. +* measObjClass matching of the class of the measObjInstId. The measObjInstId must follow the 3GPP naming conventions for Managed Objects (3GPP TS 32.106-8). + Example, for a distinguished name "ManagedElement=RNC-Gbg-1,ENodeBFunction=1", the MO class will be "ENodeBFunction". All PM filter properties are optional and a non given will result in "match all". The result of the filtering is still following the structure of a 3GPP PM report. @@ -195,7 +197,7 @@ Below follows an example of a PM filter. .. code-block:: javascript { - "filterType":"pmdata" + "filterType":"pmdata", "filter": { "sourceNames":[ "O-DU-1122" @@ -205,7 +207,7 @@ Below follows an example of a PM filter. ], "measTypes":[ "succImmediateAssignProcs" - ],eparate call. + ], "measuredEntityDns":[ "ManagedElement=RNC-Gbg-1" ] diff --git a/src/main/java/org/oran/dmaapadapter/controllers/ErrorResponse.java b/src/main/java/org/oran/dmaapadapter/controllers/ErrorResponse.java index 39f62fb..6ce5473 100644 --- a/src/main/java/org/oran/dmaapadapter/controllers/ErrorResponse.java +++ b/src/main/java/org/oran/dmaapadapter/controllers/ErrorResponse.java @@ -35,6 +35,7 @@ import reactor.core.publisher.Mono; public class ErrorResponse { private static Gson gson = new GsonBuilder() // + .disableHtmlEscaping() // .create(); // // Returned as body for all failed REST calls diff --git a/src/main/java/org/oran/dmaapadapter/controllers/ProducerCallbacksController.java b/src/main/java/org/oran/dmaapadapter/controllers/ProducerCallbacksController.java index 32ecd73..595de4a 100644 --- a/src/main/java/org/oran/dmaapadapter/controllers/ProducerCallbacksController.java +++ b/src/main/java/org/oran/dmaapadapter/controllers/ProducerCallbacksController.java @@ -61,7 +61,7 @@ public class ProducerCallbacksController { public static final String API_DESCRIPTION = ""; public static final String JOB_URL = "/generic_dataproducer/info_job"; public static final String SUPERVISION_URL = "/generic_dataproducer/health_check"; - private static Gson gson = new GsonBuilder().create(); + private static Gson gson = new GsonBuilder().disableHtmlEscaping().create(); private final Jobs jobs; private final InfoTypes types; diff --git a/src/main/java/org/oran/dmaapadapter/repository/Job.java b/src/main/java/org/oran/dmaapadapter/repository/Job.java index 90827da..d296f8e 100644 --- a/src/main/java/org/oran/dmaapadapter/repository/Job.java +++ b/src/main/java/org/oran/dmaapadapter/repository/Job.java @@ -41,7 +41,7 @@ import org.slf4j.LoggerFactory; @ToString public class Job { - private static com.google.gson.Gson gson = new GsonBuilder().create(); + private static com.google.gson.Gson gson = new GsonBuilder().disableHtmlEscaping().create(); private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static class Parameters { diff --git a/src/main/java/org/oran/dmaapadapter/repository/filters/JsonPathFilter.java b/src/main/java/org/oran/dmaapadapter/repository/filters/JsonPathFilter.java index a958649..8022312 100644 --- a/src/main/java/org/oran/dmaapadapter/repository/filters/JsonPathFilter.java +++ b/src/main/java/org/oran/dmaapadapter/repository/filters/JsonPathFilter.java @@ -29,7 +29,7 @@ public class JsonPathFilter implements Filter { private String expression; private static final Logger logger = LoggerFactory.getLogger(JsonPathFilter.class); - com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create(); + com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create(); public JsonPathFilter(String exp) { try { diff --git a/src/main/java/org/oran/dmaapadapter/repository/filters/PmReportFilter.java b/src/main/java/org/oran/dmaapadapter/repository/filters/PmReportFilter.java index 39c7a06..78334e2 100644 --- a/src/main/java/org/oran/dmaapadapter/repository/filters/PmReportFilter.java +++ b/src/main/java/org/oran/dmaapadapter/repository/filters/PmReportFilter.java @@ -32,7 +32,7 @@ import org.thymeleaf.util.StringUtils; public class PmReportFilter implements Filter { - private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create(); + private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create(); private final FilterData filterData; @Getter @@ -41,6 +41,7 @@ public class PmReportFilter implements Filter { final Collection measObjInstIds = new ArrayList<>(); final Collection measTypes = new HashSet<>(); final Collection measuredEntityDns = new ArrayList<>(); + final Collection measObjClass = new HashSet<>(); } private static class MeasTypesIndexed extends PmReport.MeasTypes { @@ -114,12 +115,38 @@ public class PmReportFilter implements Filter { return newMeasResults; } + private boolean isMeasInstIdMatch(String measObjInstId, FilterData filter) { + return filter.measObjInstIds.isEmpty() || isContainedInAny(measObjInstId, filter.measObjInstIds); + } + + private String managedObjectClass(String distinguishedName) { + int lastRdn = distinguishedName.lastIndexOf(","); + if (lastRdn == -1) { + return ""; + } + int lastEqualChar = distinguishedName.indexOf("=", lastRdn); + if (lastEqualChar == -1) { + return ""; + } + return distinguishedName.substring(lastRdn + 1, lastEqualChar); + } + + private boolean isMeasInstClassMatch(String measObjInstId, FilterData filter) { + if (filter.measObjClass.isEmpty()) { + return true; + } + + String measObjClass = managedObjectClass(measObjInstId); + return filter.measObjClass.contains(measObjClass); + } + private PmReport.MeasValuesList createMeasValuesList(PmReport.MeasValuesList oldMeasValues, PmReport.MeasTypes measTypes, FilterData filter) { PmReport.MeasValuesList newMeasValuesList = oldMeasValues.shallowClone(); - if (filter.measObjInstIds.isEmpty() || isContainedInAny(oldMeasValues.measObjInstId, filter.measObjInstIds)) { + if (isMeasInstIdMatch(oldMeasValues.measObjInstId, filter) + && isMeasInstClassMatch(oldMeasValues.measObjInstId, filter)) { newMeasValuesList.measResults = createMeasResults(oldMeasValues.measResults, measTypes, filter); } return newMeasValuesList; diff --git a/src/main/java/org/oran/dmaapadapter/tasks/DmaapTopicListener.java b/src/main/java/org/oran/dmaapadapter/tasks/DmaapTopicListener.java index fc571d5..4f20c35 100644 --- a/src/main/java/org/oran/dmaapadapter/tasks/DmaapTopicListener.java +++ b/src/main/java/org/oran/dmaapadapter/tasks/DmaapTopicListener.java @@ -44,7 +44,7 @@ public class DmaapTopicListener implements TopicListener { private final AsyncRestClient dmaapRestClient; private final ApplicationConfig applicationConfig; private final InfoType type; - private final com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create(); + private final com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create(); private Flux dataFromDmaap; public DmaapTopicListener(ApplicationConfig applicationConfig, InfoType type, SecurityContext securityContext) { diff --git a/src/main/java/org/oran/dmaapadapter/tasks/KafkaTopicListener.java b/src/main/java/org/oran/dmaapadapter/tasks/KafkaTopicListener.java index 11c0c28..3c6399f 100644 --- a/src/main/java/org/oran/dmaapadapter/tasks/KafkaTopicListener.java +++ b/src/main/java/org/oran/dmaapadapter/tasks/KafkaTopicListener.java @@ -78,10 +78,11 @@ public class KafkaTopicListener implements TopicListener { logger.error("No kafka boostrap server is setup"); } consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, this.applicationConfig.getKafkaBootStrapServers()); - consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "osc-dmaap-adapter"); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "osc-dmaap-adapter-" + this.type.getId()); consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); + consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, this.applicationConfig.getSelfUrl()); return ReceiverOptions.create(consumerProps) .subscription(Collections.singleton(this.type.getKafkaInputTopic())); diff --git a/src/main/java/org/oran/dmaapadapter/tasks/ProducerRegstrationTask.java b/src/main/java/org/oran/dmaapadapter/tasks/ProducerRegstrationTask.java index 304eb18..e0f897e 100644 --- a/src/main/java/org/oran/dmaapadapter/tasks/ProducerRegstrationTask.java +++ b/src/main/java/org/oran/dmaapadapter/tasks/ProducerRegstrationTask.java @@ -64,9 +64,8 @@ public class ProducerRegstrationTask { private final AsyncRestClient restClient; private final ApplicationConfig applicationConfig; private final InfoTypes types; - private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create(); + private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create(); - private static final String PRODUCER_ID = "DmaapGenericInfoProducer"; @Getter private boolean isRegisteredInIcs = false; private static final int REGISTRATION_SUPERVISION_INTERVAL_MS = 1000 * 10; @@ -102,10 +101,14 @@ public class ProducerRegstrationTask { logger.warn("Registration of producer failed {}", t.getMessage()); } + private String producerRegistrationUrl() { + final String producerId = this.applicationConfig.getSelfUrl().replace("/", "_"); + return applicationConfig.getIcsBaseUrl() + "/data-producer/v1/info-producers/" + producerId; + } + // Returns TRUE if registration is correct private Mono checkRegistration() { - final String url = applicationConfig.getIcsBaseUrl() + "/data-producer/v1/info-producers/" + PRODUCER_ID; - return restClient.get(url) // + return restClient.get(producerRegistrationUrl()) // .flatMap(this::isRegisterredInfoCorrect) // .onErrorResume(t -> Mono.just(Boolean.FALSE)); } @@ -126,8 +129,6 @@ public class ProducerRegstrationTask { private Mono registerTypesAndProducer() { final int CONCURRENCY = 20; - final String producerUrl = - applicationConfig.getIcsBaseUrl() + "/data-producer/v1/info-producers/" + PRODUCER_ID; return Flux.fromIterable(this.types.getAll()) // .doOnNext(type -> logger.info("Registering type {}", type.getId())) // @@ -135,7 +136,7 @@ public class ProducerRegstrationTask { CONCURRENCY) // .collectList() // .doOnNext(type -> logger.info("Registering producer")) // - .flatMap(resp -> restClient.put(producerUrl, gson.toJson(producerRegistrationInfo()))); + .flatMap(resp -> restClient.put(producerRegistrationUrl(), gson.toJson(producerRegistrationInfo()))); } private Object typeSpecifcInfoObject() { diff --git a/src/main/resources/typeSchemaPmData.json b/src/main/resources/typeSchemaPmData.json index 10c7662..5e774ce 100644 --- a/src/main/resources/typeSchemaPmData.json +++ b/src/main/resources/typeSchemaPmData.json @@ -28,6 +28,14 @@ } ] }, + "measObjClass": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + }, "measTypes": { "type": "array", "items": [ diff --git a/src/test/java/org/oran/dmaapadapter/IcsSimulatorController.java b/src/test/java/org/oran/dmaapadapter/IcsSimulatorController.java index 6d29c29..ab9e15c 100644 --- a/src/test/java/org/oran/dmaapadapter/IcsSimulatorController.java +++ b/src/test/java/org/oran/dmaapadapter/IcsSimulatorController.java @@ -56,7 +56,7 @@ import org.springframework.web.bind.annotation.RestController; public class IcsSimulatorController { private final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final static Gson gson = new GsonBuilder().create(); + private final static Gson gson = new GsonBuilder().disableHtmlEscaping().create(); public static class TestResults { diff --git a/src/test/java/org/oran/dmaapadapter/IntegrationWithIcs.java b/src/test/java/org/oran/dmaapadapter/IntegrationWithIcs.java index 9c8f816..c34d0f3 100644 --- a/src/test/java/org/oran/dmaapadapter/IntegrationWithIcs.java +++ b/src/test/java/org/oran/dmaapadapter/IntegrationWithIcs.java @@ -84,7 +84,7 @@ class IntegrationWithIcs { @Autowired private SecurityContext securityContext; - private static Gson gson = new GsonBuilder().create(); + private static Gson gson = new GsonBuilder().disableHtmlEscaping().create(); static class TestApplicationConfig extends ApplicationConfig { diff --git a/src/test/java/org/oran/dmaapadapter/IntegrationWithKafka.java b/src/test/java/org/oran/dmaapadapter/IntegrationWithKafka.java index 2de10fd..3bb3a4a 100644 --- a/src/test/java/org/oran/dmaapadapter/IntegrationWithKafka.java +++ b/src/test/java/org/oran/dmaapadapter/IntegrationWithKafka.java @@ -107,7 +107,7 @@ class IntegrationWithKafka { @Autowired private SecurityContext securityContext; - private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create(); + private static com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create(); private final Logger logger = LoggerFactory.getLogger(IntegrationWithKafka.class); @@ -448,7 +448,7 @@ class IntegrationWithKafka { @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests. @Test void kafkaCharacteristics_pmFilter() throws Exception { - // Filter PM reports and senttotowjobs over Kafka + // Filter PM reports and sent to two jobs over Kafka final String JOB_ID = "kafkaCharacteristics"; final String JOB_ID2 = "kafkaCharacteristics2"; @@ -459,7 +459,7 @@ class IntegrationWithKafka { PmReportFilter.FilterData filterData = new PmReportFilter.FilterData(); filterData.getMeasTypes().add("succImmediateAssignProcs"); - filterData.getMeasObjInstIds().add("UtranCell"); + filterData.getMeasObjClass().add("UtranCell"); this.icsSimulatorController.addJob(consumerJobInfoKafka(kafkaReceiver.OUTPUT_TOPIC, filterData), JOB_ID, restClient()); diff --git a/src/test/java/org/oran/dmaapadapter/repository/filters/PmReportFilterTest.java b/src/test/java/org/oran/dmaapadapter/repository/filters/PmReportFilterTest.java index 53da055..f4a6457 100644 --- a/src/test/java/org/oran/dmaapadapter/repository/filters/PmReportFilterTest.java +++ b/src/test/java/org/oran/dmaapadapter/repository/filters/PmReportFilterTest.java @@ -67,6 +67,21 @@ class PmReportFilterTest { assertThat(filtered).contains("Gbg-997").doesNotContain("Gbg-998"); } + @Test + void testMeasObjClass() throws Exception { + PmReportFilter.FilterData filterData = new PmReportFilter.FilterData(); + filterData.measObjClass.add("junk"); + PmReportFilter filter = new PmReportFilter(filterData); + String filtered = filter.filter(loadReport()); + assertThat(filtered).isEmpty(); + + filterData = new PmReportFilter.FilterData(); + filterData.measObjClass.add("ENodeBFunction"); + filter = new PmReportFilter(filterData); + filtered = filter.filter(loadReport()); + assertThat(filtered).contains("ENodeBFunction").doesNotContain("UtranCell"); + } + @Test void testSourceNames() throws Exception { PmReportFilter.FilterData filterData = new PmReportFilter.FilterData(); @@ -99,7 +114,7 @@ class PmReportFilterTest { @Test void testParse() throws Exception { - com.google.gson.Gson gson = new com.google.gson.GsonBuilder().create(); + com.google.gson.Gson gson = new com.google.gson.GsonBuilder().disableHtmlEscaping().create(); PmReport report = gson.fromJson(loadReport(), PmReport.class); String dn = report.event.perf3gppFields.measDataCollection.measuredEntityDn; -- 2.16.6