diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 3cee82d8b..59b5a5125 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ --[cygnus-ngsi][GenericRowAggregator] Fix generic row aggregator name used in logs --[cygnus-commons] Upgrade httpclient dependency from 4.3.6 to 4.5.13 due to security vulnerability +-[cygnus-ngsi][NGSINameMappingsInterceptor] Now JsonPath expressions are allowed while mapping attributes, Interceptor can be configured to don't stop on first match. (#1856) +-[cygnus-ngsi][GenericRowAggregator] Fix generic row aggregator name used in logs +-[cygnus-commons] Upgrade httpclient dependency from 4.3.6 to 4.5.13 due to security vulnerability \ No newline at end of file diff --git a/cygnus-common/pom.xml b/cygnus-common/pom.xml index 721627f51..ab052139f 100644 --- a/cygnus-common/pom.xml +++ b/cygnus-common/pom.xml @@ -16,6 +16,12 @@ + + + com.jayway.jsonpath + json-path + 2.4.0 + org.mockito @@ -144,6 +150,7 @@ true + diff --git a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/containers/NameMappings.java b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/containers/NameMappings.java index 39eb728bf..3c4610551 100644 --- a/cygnus-common/src/main/java/com/telefonica/iot/cygnus/containers/NameMappings.java +++ b/cygnus-common/src/main/java/com/telefonica/iot/cygnus/containers/NameMappings.java @@ -18,8 +18,17 @@ package com.telefonica.iot.cygnus.containers; import java.util.ArrayList; +import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.PathNotFoundException; import com.telefonica.iot.cygnus.log.CygnusLogger; /** @@ -27,22 +36,23 @@ * @author frb */ public class NameMappings { - + private static final String DEFAULT_ORIGINAL_MAPPING = "^(.*)"; private final ArrayList serviceMappings; - + private static final CygnusLogger LOGGER = new CygnusLogger(NameMappings.class); + /** * Constructor. */ public NameMappings() { serviceMappings = new ArrayList<>(); } // NameMappings - + public ArrayList getServiceMappings() { return serviceMappings; } // getServiceMappings - + /** * Purges the Name Mappings if any field is missing or has an invalid value. */ @@ -53,7 +63,7 @@ public void purge() { } // for } // if } // purge - + /** * Compiles the regular expressions into Java Patterns. */ @@ -64,15 +74,16 @@ public void compilePatterns() { } // for } // if } // compilePatterns - + /** * Overwrite of toString() method. + * * @return */ @Override public String toString() { String nameMappingsStr = "{\"serviceMappings\":["; - + if (serviceMappings != null) { boolean first = true; @@ -85,103 +96,106 @@ public String toString() { } // if eslse } // for } // if - + nameMappingsStr += "]}"; return nameMappingsStr; } // toString - + /** * Adds new service mappings to these name mappings. + * * @param newServiceMappings * @param update */ public void add(ArrayList newServiceMappings, boolean update) { for (ServiceMapping newServiceMapping : newServiceMappings) { ServiceMapping serviceMapping = get(newServiceMapping.originalService); - + if (serviceMapping == null) { serviceMappings.add(newServiceMapping); } else { if (update) { serviceMapping.newService = newServiceMapping.newService; } // if - + serviceMapping.add(newServiceMapping.getServicePathMappings(), update); } // if else } // for } // add - + /** * Removes service mappings from these name mappings if there are no service path mappings. + * * @param newServiceMappings */ public void remove(ArrayList newServiceMappings) { for (ServiceMapping newServiceMapping : newServiceMappings) { ServiceMapping serviceMapping = get(newServiceMapping.originalService); - + if (serviceMapping != null) { - ArrayList newServicePathMappings = newServiceMapping.getServicePathMappings(); - + ArrayList newServicePathMappings = newServiceMapping + .getServicePathMappings(); + if (newServicePathMappings == null) { serviceMappings.remove(serviceMapping); } else { serviceMapping.remove(newServicePathMappings); } // if else } // if - // else { - // Nothing is done if the service mapping is null - // } + // else { + // Nothing is done if the service mapping is null + // } } // for } // remove - + private ServiceMapping get(String originalService) { for (ServiceMapping serviceMapping : serviceMappings) { if (serviceMapping.originalService.equals(originalService)) { return serviceMapping; } // if } // for - + return null; } // get - + /** * ServiceMapping class. */ public class ServiceMapping { - + private String originalService; private Pattern originalServicePattern; private String newService; private final ArrayList servicePathMappings; - + /** * Constructor. */ public ServiceMapping() { servicePathMappings = new ArrayList<>(); } // NameMappings - + public ArrayList getServicePathMappings() { return servicePathMappings; } // getServicePathMappings - + public String getOriginalService() { return originalService; } // getOriginalService - + public String getNewService() { return newService; } // getNewService - + public Pattern getOriginalServicePattern() { return originalServicePattern; } // getOriginalServicePattern - + /** * Purges the ServiceMappings if any field is missing or has an invalid value. */ public void purge() { - if (originalService == null ) { + if (originalService == null) { originalService = DEFAULT_ORIGINAL_MAPPING; LOGGER.debug("[NameMappings] No originalService found in mapping, using default."); } @@ -191,7 +205,7 @@ public void purge() { } // for } // if } // purge - + /** * Compiles the regular expressions into Java Patterns. */ @@ -204,14 +218,12 @@ public void compilePatterns() { } // for } // if } // compilePatterns - + @Override public String toString() { - String serviceMappingStr = - "{\"originalService\":\"" + getOriginalService() + "\"," - + "\"newService\":\"" + getNewService() + "\"," - + "\"servicePathMappings\":["; - + String serviceMappingStr = "{\"originalService\":\"" + getOriginalService() + "\"," + + "\"newService\":\"" + getNewService() + "\"," + "\"servicePathMappings\":["; + if (servicePathMappings != null) { boolean first = true; @@ -224,19 +236,21 @@ public String toString() { } // if else } // for } // if - + serviceMappingStr += "]}"; return serviceMappingStr; } // toString - + /** * Adds new service path mappings to this service mapping. + * * @param newServicePathMappings * @param update */ public void add(ArrayList newServicePathMappings, boolean update) { for (ServicePathMapping newServicePathMapping : newServicePathMappings) { - ServicePathMapping servicePathMapping = get(newServicePathMapping.originalServicePath); + ServicePathMapping servicePathMapping = get( + newServicePathMapping.originalServicePath); if (servicePathMapping == null) { servicePathMappings.add(newServicePathMapping); @@ -244,22 +258,26 @@ public void add(ArrayList newServicePathMappings, boolean up if (update) { servicePathMapping.newServicePath = newServicePathMapping.newServicePath; } // if - + servicePathMapping.add(newServicePathMapping.getEntityMappings(), update); } // if else } // for } // add - + /** - * Removes service path mappings from these service mappings if there are no entity mappings. + * Removes service path mappings from these service mappings if there are no entity + * mappings. + * * @param newServicePathMappings */ public void remove(ArrayList newServicePathMappings) { for (ServicePathMapping newServicePathMapping : newServicePathMappings) { - ServicePathMapping servicePathMapping = get(newServicePathMapping.originalServicePath); + ServicePathMapping servicePathMapping = get( + newServicePathMapping.originalServicePath); if (servicePathMapping != null) { - ArrayList newEntityMappings = newServicePathMapping.getEntityMappings(); + ArrayList newEntityMappings = newServicePathMapping + .getEntityMappings(); if (newEntityMappings == null) { servicePathMappings.remove(servicePathMapping); @@ -267,12 +285,12 @@ public void remove(ArrayList newServicePathMappings) { servicePathMapping.remove(newEntityMappings); } // if else } // if - // else { - // Nothing is done if the service path mapping is null - // } + // else { + // Nothing is done if the service path mapping is null + // } } // for } // remove - + private ServicePathMapping get(String originalServicePath) { for (ServicePathMapping servicePathMapping : servicePathMappings) { if (servicePathMapping.originalServicePath.equals(originalServicePath)) { @@ -282,49 +300,50 @@ private ServicePathMapping get(String originalServicePath) { return null; } // get - + } // ServiceMapping - + /** * ServicePathMapping class. */ public class ServicePathMapping { - + private String originalServicePath; private Pattern originalServicePathPattern; private String newServicePath; private final ArrayList entityMappings; - + /** * Constructor. */ public ServicePathMapping() { entityMappings = new ArrayList<>(); } // ServicePathMapping - + public ArrayList getEntityMappings() { return entityMappings; } // getEntityMappings - + public String getOriginalServicePath() { return originalServicePath; } // getOriginalServicePath - + public String getNewServicePath() { return newServicePath; } // getNewServicePath - + public Pattern getOriginalServicePathPattern() { return originalServicePathPattern; } // getOriginalServicePathPattern - + /** * Purges the ServicePathMappings if any field is missing or has an invalid value. */ public void purge() { - if (originalServicePath == null ) { + if (originalServicePath == null) { originalServicePath = DEFAULT_ORIGINAL_MAPPING; - LOGGER.debug("[NameMappings] No originalServicePath found in mapping, using default."); + LOGGER.debug( + "[NameMappings] No originalServicePath found in mapping, using default."); } if (entityMappings != null) { for (EntityMapping entityMapping : entityMappings) { @@ -332,7 +351,7 @@ public void purge() { } // for } // if } // purge - + /** * Compiles the regular expressions into Java Patterns. */ @@ -345,18 +364,17 @@ public void compilePatterns() { } // for } // if } // compilePatterns - + @Override public String toString() { - String entityMappingStr = - "{\"originalServicePath\":\"" + getOriginalServicePath() + "\"," - + "\"newServicePath\":\"" + getNewServicePath() + "\"," + String entityMappingStr = "{\"originalServicePath\":\"" + getOriginalServicePath() + + "\"," + "\"newServicePath\":\"" + getNewServicePath() + "\"," + "\"entityMappings\": ["; - + if (entityMappings != null) { boolean first = true; - for (Object entityMapping: entityMappings) { + for (Object entityMapping : entityMappings) { if (first) { entityMappingStr += entityMapping.toString(); first = false; @@ -365,13 +383,14 @@ public String toString() { } // if else } // for } // if - + entityMappingStr += "]}"; return entityMappingStr; } // toString - + /** * Adds new entity mappings to this service path mapping. + * * @param newEntityMappings * @param update */ @@ -387,14 +406,16 @@ public void add(ArrayList newEntityMappings, boolean update) { entityMapping.newEntityId = newEntityMapping.newEntityId; entityMapping.newEntityType = newEntityMapping.newEntityType; } // if - + entityMapping.add(newEntityMapping.getAttributeMappings(), update); } // if else } // for } // add - + /** - * Removes entity mappings from these service path mappings if there are no attribute mappings. + * Removes entity mappings from these service path mappings if there are no attribute + * mappings. + * * @param newEntityMappings */ public void remove(ArrayList newEntityMappings) { @@ -403,7 +424,8 @@ public void remove(ArrayList newEntityMappings) { newEntityMapping.originalEntityType); if (entityMapping != null) { - ArrayList newAttributeMappings = newEntityMapping.getAttributeMappings(); + ArrayList newAttributeMappings = newEntityMapping + .getAttributeMappings(); if (newAttributeMappings == null) { entityMappings.remove(entityMapping); @@ -411,12 +433,12 @@ public void remove(ArrayList newEntityMappings) { entityMapping.remove(newAttributeMappings); } // if else } // if - // else { - // Nothing is done if the entity mapping is null - // } + // else { + // Nothing is done if the entity mapping is null + // } } // for } // remove - + private EntityMapping get(String originalEntityId, String originalEntityType) { for (EntityMapping entityMapping : entityMappings) { if (entityMapping.originalEntityId.equals(originalEntityId) @@ -427,14 +449,14 @@ private EntityMapping get(String originalEntityId, String originalEntityType) { return null; } // get - + } // ServicePathMapping - + /** * EntityMapping class. */ public class EntityMapping { - + private String originalEntityId; private Pattern originalEntityIdPattern; private String originalEntityType; @@ -442,57 +464,58 @@ public class EntityMapping { private String newEntityId; private String newEntityType; private final ArrayList attributeMappings; - + /** * Constructor. */ public EntityMapping() { - attributeMappings = new ArrayList<>(); + attributeMappings = new ArrayList(); } // EntityMapping - + public ArrayList getAttributeMappings() { return attributeMappings; } // getAttributeMappings - + public String getOriginalEntityId() { return originalEntityId; } // getOriginalEntityId - + public String getOriginalEntityType() { return originalEntityType; } // getOriginalEntityType - + public String getNewEntityId() { return newEntityId; } // getNewEntityId - + public String getNewEntityType() { return newEntityType; } // getNewEntityType - + public Pattern getOriginalEntityIdPattern() { return originalEntityIdPattern; } // getOriginalEntityIdPattern - + public Pattern getOriginalEntityTypePattern() { return originalEntityTypePattern; } // getOriginalEntityTypePattern - + /** * Purges the EntityMappings if any field is missing or has an invalid value. */ public void purge() { - if (originalEntityId == null ) { + if (originalEntityId == null) { originalEntityId = DEFAULT_ORIGINAL_MAPPING; LOGGER.debug("[NameMappings] No originalEntityId found in mapping, using default."); } - if (originalEntityType == null ) { + if (originalEntityType == null) { originalEntityType = DEFAULT_ORIGINAL_MAPPING; - LOGGER.debug("[NameMappings] No originalEntityType found in mapping, using default."); + LOGGER.debug( + "[NameMappings] No originalEntityType found in mapping, using default."); } - //TODO purge attributes + // TODO purge attributes } // purge - + /** * Compiles the regular expressions into Java Patterns. */ @@ -506,20 +529,18 @@ public void compilePatterns() { } // for } // if } // compilePatterns - + @Override public String toString() { - String attrMappingStr = - "{\"originalEntityId\":\"" + getOriginalEntityId() + "\"," + String attrMappingStr = "{\"originalEntityId\":\"" + getOriginalEntityId() + "\"," + "\"originalEntityType\":\"" + getOriginalEntityType() + "\"," - + "\"newEntityId\":\"" + getNewEntityId() + "\"," - + "\"newEntityType\":\"" + getNewEntityType() + "\"," - + "\"attributeMappings\":["; + + "\"newEntityId\":\"" + getNewEntityId() + "\"," + "\"newEntityType\":\"" + + getNewEntityType() + "\"," + "\"attributeMappings\":["; if (attributeMappings != null) { boolean first = true; - - for (Object attrMap: attributeMappings) { + + for (Object attrMap : attributeMappings) { if (first) { attrMappingStr += attrMap.toString(); first = false; @@ -528,13 +549,14 @@ public String toString() { } // if else } // for } // for - + attrMappingStr += "]}"; return attrMappingStr; } // toString - + /** * Adds new attribute mappings to this entity mapping. + * * @param newAttributeMappings * @param update */ @@ -549,14 +571,16 @@ public void add(ArrayList newAttributeMappings, boolean update attributeMapping.newAttributeName = newAttributeMapping.newAttributeName; attributeMapping.newAttributeType = newAttributeMapping.newAttributeType; } // if else - // else { - // Nothing is done if the attribute mapping already exists and updating is not enabled - // } + // else { + // Nothing is done if the attribute mapping already exists and updating is not + // enabled + // } } // for } // add - + /** * Removes attribute mappings from these entity mappings. + * * @param newAttributeMappings */ public void remove(ArrayList newAttributeMappings) { @@ -567,12 +591,12 @@ public void remove(ArrayList newAttributeMappings) { if (attributeMapping != null) { attributeMappings.remove(attributeMapping); } // if - // else { - // Nothing is done if the attribute mapping is null - // } + // else { + // Nothing is done if the attribute mapping is null + // } } // for } // remove - + private AttributeMapping get(String originalAttributeName, String originalAttributeType) { for (AttributeMapping attributeMapping : attributeMappings) { if (attributeMapping.originalAttributeName.equals(originalAttributeName) @@ -583,13 +607,15 @@ private AttributeMapping get(String originalAttributeName, String originalAttrib return null; } // get - + } // EntityMapping - + /** * AttributeMapping class. */ public class AttributeMapping { + + private static final String JSONPATH_PATTERN_STR = "^(\\w*[^\\\\])\\.([^*+?].+)"; private String originalAttributeName; private Pattern originalAttributeNamePattern; @@ -597,55 +623,108 @@ public class AttributeMapping { private Pattern originalAttributeTypePattern; private String newAttributeName; private String newAttributeType; - + private String jsonPath; + private boolean isJsonPath; + /** * Constructor. */ public AttributeMapping() { + isJsonPath = false; } // AttributeMapping - + public String getOriginalAttributeName() { return originalAttributeName; } // getOriginalAttributeName - + public String getOriginalAttributeType() { return originalAttributeType; } // getOriginalAttributeType - + public String getNewAttributeName() { return newAttributeName; } // getNewAttributeName - + public String getNewAttributeType() { return newAttributeType; } // getNewAttributeType - + public Pattern getOriginalAttributeNamePattern() { return originalAttributeNamePattern; } // getOriginalAttributeNamePattern - + public Pattern getOriginalAttributeTypePattern() { return originalAttributeTypePattern; } // getOriginalAttributeTypePattern - + + public boolean isJsonPath() { + return isJsonPath; + } // isJsonPath + + /** + * Maps value if it's JsonPath mapping. + * + * @param originalValue + * @return + */ + public JsonElement getMappedValue(JsonElement originalValue) { + JsonElement result = originalValue; + + if (isJsonPath() && originalValue != null && originalValue.isJsonObject()) { + String attValue = originalValue.toString(); + DocumentContext attContext = JsonPath.parse(attValue); + + try { + String resultValue = attContext.read(jsonPath); + LOGGER.debug( + "[NameMappings] mappedValue: " + newAttributeName + ": " + resultValue); + result = new JsonPrimitive(resultValue); + } catch (JsonParseException | PathNotFoundException e) { + LOGGER.error("[NameMappings] Unable to map attribute: " + originalAttributeName + + " jsonPath: " + jsonPath); + LOGGER.error("[NameMappings] From Json value: " + attValue); + LOGGER.error("[NameMappings] Error: " + e.getMessage()); + result = JsonNull.INSTANCE; + } + } + + return result; + } + /** * Compiles the regular expressions into Java Patterns. */ public void compilePatterns() { - originalAttributeNamePattern = Pattern.compile(originalAttributeName); + originalAttributeTypePattern = Pattern.compile(originalAttributeType); + + Pattern isJsonPathPattern = Pattern.compile(JSONPATH_PATTERN_STR); + + Matcher jsonPathMatcher = isJsonPathPattern.matcher(originalAttributeName); + isJsonPath = jsonPathMatcher.matches(); + + if (isJsonPath) { + LOGGER.debug("[NameMappings] Compile Attribute mapping patterns as JsonPath"); + originalAttributeNamePattern = Pattern.compile(jsonPathMatcher.group(1)); + jsonPath = "$." + jsonPathMatcher.group(2); + + LOGGER.debug("[NameMappings] Original att name: " + jsonPathMatcher.group(1)); + LOGGER.debug("[NameMappings] JsonPath expression: " + jsonPath); + } else { + originalAttributeNamePattern = Pattern.compile(originalAttributeName); + } + } // compilePatterns - + @Override public String toString() { - String attrMappingStr = - "{\"originalAttributeName\":\"" + getOriginalAttributeName() + "\"," - + "\"originalAttributeType\":\"" + getOriginalAttributeType() + "\"," + String attrMappingStr = "{\"originalAttributeName\":\"" + getOriginalAttributeName() + + "\"," + "\"originalAttributeType\":\"" + getOriginalAttributeType() + "\"," + "\"newAttributeName\":\"" + getNewAttributeName() + "\"," + "\"newAttributeType\":\"" + getNewAttributeType() + "\"}"; return attrMappingStr; } // toString - + } // AttributeMapping - + } // NameMappings diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptor.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptor.java index b4e86c925..0cb6470b6 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptor.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptor.java @@ -17,7 +17,18 @@ */ package com.telefonica.iot.cygnus.interceptors; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.flume.Context; +import org.apache.flume.Event; +import org.apache.flume.interceptor.Interceptor; + import com.google.gson.Gson; +import com.google.gson.JsonElement; import com.google.gson.JsonIOException; import com.google.gson.JsonSyntaxException; import com.telefonica.iot.cygnus.containers.NameMappings; @@ -31,14 +42,6 @@ import com.telefonica.iot.cygnus.utils.CommonConstants; import com.telefonica.iot.cygnus.utils.JsonUtils; import com.telefonica.iot.cygnus.utils.NGSIConstants; -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.flume.Context; -import org.apache.flume.Event; -import org.apache.flume.interceptor.Interceptor; /** * @@ -48,6 +51,7 @@ public class NGSINameMappingsInterceptor implements Interceptor { private static final CygnusLogger LOGGER = new CygnusLogger(NGSINameMappingsInterceptor.class); private final String nameMappingsConfFile; + private final boolean stopOnFirstAttrMatch; private final boolean invalidConfiguration; private NameMappings nameMappings; private PeriodicalNameMappingsReader periodicalNameMappingsReader; @@ -56,13 +60,26 @@ public class NGSINameMappingsInterceptor implements Interceptor { * Constructor. * * @param nameMappingsConfFile + * @param stopOnFirstAttrMatch * @param invalidConfiguration */ - public NGSINameMappingsInterceptor(String nameMappingsConfFile, boolean invalidConfiguration) { + public NGSINameMappingsInterceptor(String nameMappingsConfFile, boolean stopOnFirstAttrMatch, + boolean invalidConfiguration) { this.nameMappingsConfFile = nameMappingsConfFile; + this.stopOnFirstAttrMatch = stopOnFirstAttrMatch; this.invalidConfiguration = invalidConfiguration; } // NGSINameMappingsInterceptor + /** + * Constructor. + * + * @param nameMappingsConfFile + * @param invalidConfiguration + */ + public NGSINameMappingsInterceptor(String nameMappingsConfFile, boolean invalidConfiguration) { + this(nameMappingsConfFile, true, invalidConfiguration); + } // NGSINameMappingsInterceptor + @Override public void initialize() { if (!invalidConfiguration) { @@ -150,10 +167,12 @@ public void close() { public static class Builder implements Interceptor.Builder { private boolean invalidConfiguration; private String nameMappingsConfFile; + private Boolean stopOnFirstAttrMatch; @Override public void configure(Context context) { nameMappingsConfFile = context.getString("name_mappings_conf_file"); + stopOnFirstAttrMatch = context.getBoolean("stop_on_first_attr_match", true); if (nameMappingsConfFile == null) { invalidConfiguration = true; @@ -168,7 +187,7 @@ public void configure(Context context) { @Override public Interceptor build() { - return new NGSINameMappingsInterceptor(nameMappingsConfFile, invalidConfiguration); + return new NGSINameMappingsInterceptor(nameMappingsConfFile, stopOnFirstAttrMatch, invalidConfiguration); } // build protected boolean getInvalidConfiguration() { @@ -178,8 +197,7 @@ protected boolean getInvalidConfiguration() { } // Builder /** - * Class in charge or periodically reading the NGSINameMappingsInterceptor - * configuration file. + * Class in charge or periodically reading the NGSINameMappingsInterceptor configuration file. */ private class PeriodicalNameMappingsReader extends Thread { @@ -250,8 +268,8 @@ private void loadNameMappings() { } // loadNameMappings /** - * Loads the Name Mappings given a Json string. It is protected since it - * only can be used by this class and test classes. + * Loads the Name Mappings given a Json string. It is protected since it only can be used by this class and test + * classes. * * @param jsonStr */ @@ -303,7 +321,7 @@ protected void loadNameMappings(String jsonStr) { * @param originalCE * @return The input NotifyContextRequest object with maps applied */ - public ImmutableTriple doMap(String originalService, String originalServicePath, + public ImmutableTriple doMap(String originalService,String originalServicePath, ContextElement originalCE) { if (nameMappings == null) { LOGGER.info("[nmi] No namemappings to map entity " + originalCE.toString()); @@ -330,7 +348,7 @@ public ImmutableTriple doMap(String originalServ if (serviceMapping.getNewService() != null) { newService = originalService.replaceAll(serviceMapping.getOriginalServicePattern().toString(), - serviceMapping.getNewService()); + serviceMapping.getNewService()); LOGGER.debug("[nmi] FIWARE new service obtained: " + newService); } // if @@ -361,8 +379,8 @@ public ImmutableTriple doMap(String originalServ continue; } else { if (spm.getNewServicePath() != null) { - newServicePath = originalServicePath.replaceAll(spm.getOriginalServicePathPattern().toString(), - spm.getNewServicePath()); + newServicePath = originalServicePath.replaceAll( + spm.getOriginalServicePathPattern().toString(), spm.getNewServicePath()); } LOGGER.debug("[nmi] FIWARE new service path obtained: " + newServicePath); servicePathMapping = spm; @@ -371,7 +389,7 @@ public ImmutableTriple doMap(String originalServ } else { if (spm.getNewServicePath() != null) { newServicePath = originalServicePath.replaceAll(spm.getOriginalServicePathPattern().toString(), - spm.getNewServicePath()); + spm.getNewServicePath()); } LOGGER.debug("[nmi] FIWARE new service path obtained: " + newServicePath); servicePathMapping = spm; @@ -416,8 +434,8 @@ public ImmutableTriple doMap(String originalServ LOGGER.debug("[nmi] " + entityMapping.getOriginalEntityId() + " matches " + newCE.getId()); } } - if (!entityMapping.getOriginalEntityIdPattern().matcher(originalEntityId).matches() || - !entityMapping.getOriginalEntityTypePattern().matcher(originalEntityType).matches()) { + if (!entityMapping.getOriginalEntityIdPattern().matcher(originalEntityId).matches() + || !entityMapping.getOriginalEntityTypePattern().matcher(originalEntityType).matches()) { LOGGER.debug("[nmi] not matches both type and entityId"); entityMapping = null; continue; @@ -446,15 +464,20 @@ public ImmutableTriple doMap(String originalServ newCE.setId(newEntityId); newCE.setType(newEntityType); + List newAttributes = new ArrayList(); for (ContextAttribute newCA : newCE.getAttributes()) { LOGGER.debug("[nmi] checking with CA: " + newCA.toString()); + JsonElement originalCAValue = newCA.getValue(); String originalAttributeName = newCA.getName(); String originalAttributeType = newCA.getType(); String newAttributeName = originalAttributeName; String newAttributeType = originalAttributeType; AttributeMapping attributeMapping = null; + boolean firstMatch = true; + boolean attributeFound = false; + for (AttributeMapping am : entityMapping.getAttributeMappings()) { attributeMapping = am; LOGGER.debug("[nmi] checking with attributeMapping: " + attributeMapping.toString()); @@ -478,14 +501,16 @@ public ImmutableTriple doMap(String originalServ } } - if (!attributeMapping.getOriginalAttributeNamePattern().matcher(originalAttributeName).matches() || - !attributeMapping.getOriginalAttributeTypePattern().matcher(originalAttributeType).matches()) { + if (!attributeMapping.getOriginalAttributeNamePattern().matcher(originalAttributeName).matches() + || !attributeMapping.getOriginalAttributeTypePattern().matcher(originalAttributeType) + .matches()) { LOGGER.debug("[nmi] not matches both attribute type and name"); attributeMapping = null; continue; } // if LOGGER.debug("[nmi] Attribute found: " + originalAttributeName + ", " + originalAttributeType); + attributeFound = true; if (attributeMapping.getNewAttributeName() != null) { newAttributeName = attributeMapping.getNewAttributeName(); @@ -495,19 +520,43 @@ public ImmutableTriple doMap(String originalServ newAttributeType = attributeMapping.getNewAttributeType(); } // if - break; + // Modify context data, or add a new attribute to the list. + if (firstMatch) { + newCA.setName(newAttributeName); + newCA.setType(newAttributeType); + newCA.setContextValue(attributeMapping.getMappedValue(newCA.getValue())); + LOGGER.debug("[nmi] newCA: " + newCA.toString()); + + if (this.stopOnFirstAttrMatch) { + break; + } else { + firstMatch = false; + } + } else { + ContextAttribute otherCA = newCA.deepCopy(); + otherCA.setName(newAttributeName); + otherCA.setType(newAttributeType); + otherCA.setContextValue(attributeMapping.getMappedValue(originalCAValue)); + LOGGER.debug("[nmi] brand newCA: " + otherCA.toString()); + + newAttributes.add(otherCA); + } + } // for - if (attributeMapping == null) { + if (!attributeFound) { LOGGER.debug("[nmi] Attribute not found: " + originalAttributeName + ", " + originalAttributeType); - continue; - } // if + } - newCA.setName(newAttributeName); - newCA.setType(newAttributeType); - LOGGER.debug("[nmi] newCA: " + newCA.toString()); } // for + + // Add new Attributes + if (newAttributes.size() > 0) { + newCE.getAttributes().addAll(newAttributes); + } + LOGGER.info("[nmi] Entity " + originalCE.toString() + " mapped to: " + newCE.toString() + " by matched with " + entityMapping.toString()); + return new ImmutableTriple(newService, newServicePath, newCE); } // map diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/containers/NameMappingsTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/containers/NameMappingsTest.java index 8c18c4251..c447e4f3c 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/containers/NameMappingsTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/containers/NameMappingsTest.java @@ -249,7 +249,7 @@ public class NameMappingsTest { + " \"newServicePath\": \"/new_default\"," + " \"entityMappings\": [" + " {" - + " \"originalEntityId\": \"Room\\.(\\d*)\"," + + " \"originalEntityId\": \"Room([0-9]*)\"," + " \"originalEntityType\": \"Room\"," + " \"newEntityId\": \"new_Room1\"," + " \"newEntityType\": \"new_Room\"," diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptorTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptorTest.java index 8504c91a6..4e7a3bdcc 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptorTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/interceptors/NGSINameMappingsInterceptorTest.java @@ -24,6 +24,8 @@ import com.telefonica.iot.cygnus.utils.NGSIConstants; import com.telefonica.iot.cygnus.utils.NGSIUtilsForTests; import java.util.Map; +import java.util.regex.Pattern; + import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.flume.Context; import org.apache.log4j.Level; @@ -743,7 +745,7 @@ public void testDoMapConfig4() { System.out.println(getTestTraceHead("[NGSINameMappingInterceptor.doMapConfig4]") + "- ERROR - The Service type is not equal to the expected one"); equals = false; - } + } try { assertTrue(equals); @@ -756,7 +758,6 @@ public void testDoMapConfig4() { } // try catch } // testDoMapConfig4 - /* * [NGSINameMappingsInterceptorTest.loadNameMappingsError] -------- Load a name mappings with error. * @throws java.lang.Exception diff --git a/doc/cygnus-ngsi/installation_and_administration_guide/name_mappings.md b/doc/cygnus-ngsi/installation_and_administration_guide/name_mappings.md index 1acf08294..67f9ebffe 100644 --- a/doc/cygnus-ngsi/installation_and_administration_guide/name_mappings.md +++ b/doc/cygnus-ngsi/installation_and_administration_guide/name_mappings.md @@ -136,9 +136,13 @@ $ cat /path/to/conf/name_mappings.conf } ``` -Last but not least, the original names support Java-based regular expressions. For instance, if you want all the service paths are re-named equals simply use `/.*` as value for `originalServicePath`: +## Advanced Mapping -``` +### Regular Expresion Matching + +The original names support Java-based regular expressions. For instance, if you want all the service paths are re-named equals simply use `/.*` as value for `originalServicePath`: + +```json $ cat /path/to/conf/name_mappings.conf { "serviceMappings": [ @@ -171,9 +175,13 @@ $ cat /path/to/conf/name_mappings.conf ] } ``` + + + +### Find and replace using regular expressions In addition, above mentioned Java-based regular expressions can be also used in new entity IDs. For instance, if we want to rename certain entity IDs described as a string concatenated with a number, and we want to replace the string but maintaining the number: -``` +```json { "serviceMappings": [{ "originalService": "service", @@ -191,9 +199,52 @@ In addition, above mentioned Java-based regular expressions can be also used in }] } ``` - +### Clone attributes +By default, `NGSIMappingInterceptor` stops seraching map rules for an attribute when it finds a match for it. We may need an attribute to match more than one mapping rule for some reasons. This can be done by setting `stop_on_first_attr_match` interceptor's property to false inside agent configuration file. +``` +agent.sources.ngsi-source.interceptors = nmi +agent.sources.ngsi-source.interceptors.nmi.type = com.telefonica.iot.cygnus.interceptors.NGSINameMappingsInterceptor$Builder +agent.sources.ngsi-source.interceptors.nmi.name_mappings_conf_file = /path/name_mappings.conf +agent.sources.ngsi-source.interceptors.nmi.stop_on_first_attr_match = false +``` +This option has an impact on performance, so it should only be used where necessary. + +### Using JsonPath Expresions +For attributes, as an alternative to regular expressions, we can use JsonPath based expressions. +This allows us to obtain a primitive data contained within a complex NGSI attribute. + +Postal address `NGSI` attributes are a good example. We can use JsonPath if we need to persist the __city__ and the __address__ individually. +```json +"address": { + "type": "PostalAddress", + "addressLocality": "Santander", + "addressRegion": "CA", + "postalCode": "39002", + "streetAddress": "39002 Plaza Ayuntamiento 1" + } +``` +Name mapping should look like: +```json +"attributeMappings": [ + { + "originalAttributeName": "address.addressLocality", + "newAttributeName": "city", + ... + }, + { + "originalAttributeName": "address.streetAddress", + "newAttributeName": "street", + ... + }, + ... +] +``` +Note that `JsonPath` expression must start with the original attribute name to allow `NGSIMappingInterceptor` to do the match. + +Remark that as we need to split the source attribute, `stop_on_first_attr_match` interceptor's property must be set to __false__, otherwhise only city attribute will be generated. + +## Examples Sumarizing these are some useful examples and their result in a sink like MySQL: - ### Case 1: groups matching exactly by service, subservice, entityid and entitytype - Service Mapping: ```