Use unique labels for indexing within multiple OSM layers

Essentially when adding nodes of a particular label, we also add a label with a unique name for indexing purposes.
The unique name is made of the original name plus a hex suffix made of the MD5 hash of the layer name.

For example, the layer `geom1` will have an MD5 hash of its name `9ECE5459EA0D46FC556E5E3F454A0795`.
Then when adding an OSM node we label the node with both:

* OSMNode
* OSMNode_9ECE5459EA0D46FC556E5E3F454A0795

The second label is also used to create a label-property index to be used for looking up the node when building ways.

Author: craigtaverner

README.md

@@ -74,6 +74,13 @@ This has meant that the spatial library needed a major refactoring to work with
   and simply add on the rights to create tokens and indexes. In 0.27.2 we instead use `RestrictedAccessMode`
   to restrict the users access right to the built in `AccessModel.Static.SCHEMA` and then boost to enable
   index and token writes. The difference is subtle and should only be possible to notice in Enterprise Edition.
+* 0.28.0 tackles the ability to import multiple OSM files. The initial solution for Neo4j 4.x made use
+  of schema indexes keyed by the label and property. However, that means that all OSM imports would share
+  the same index. If they are completely disjointed data sets, this would not matter. But if you import
+  overlapping OSM files or different versions of the same file file, a mangled partial merger would result.
+  0.28.0 solves this by using different indexes, and keeping all imports completely separate.
+  The more complex problems of importing newer versions, and stitching together overlapping areas, are not
+  yet solved.
 
 Consequences of the port to Neo4j 4.x:
 
@@ -347,6 +354,7 @@ The Neo4j Spatial Plugin is available for inclusion in the server version of Neo
   * [v0.27.0 for Neo4j 4.0.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.0-neo4j-4.0.3/neo4j-spatial-0.27.0-neo4j-4.0.3-server-plugin.jar?raw=true)
   * [v0.27.1 for Neo4j 4.1.7](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.1-neo4j-4.1.7/neo4j-spatial-0.27.1-neo4j-4.1.7-server-plugin.jar?raw=true)
   * [v0.27.2 for Neo4j 4.2.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.2-neo4j-4.2.3/neo4j-spatial-0.27.2-neo4j-4.2.3-server-plugin.jar?raw=true)
+  * [v0.28.0 for Neo4j 4.2.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.28.0-neo4j-4.2.3/neo4j-spatial-0.28.0-neo4j-4.2.3-server-plugin.jar?raw=true)
 
 For versions up to 0.15-neo4j-2.3.4:
 
@@ -463,7 +471,7 @@ Add the following repositories and dependency to your project's pom.xml:
     <dependency>
         <groupId>org.neo4j</groupId>
         <artifactId>neo4j-spatial</artifactId>
-        <version>0.27.2-neo4j-4.2.3</version>
+        <version>0.28.0-neo4j-4.2.3</version>
     </dependency>
 ~~~
 

pom.xml

@@ -23,7 +23,7 @@
     <modelVersion>4.0.0</modelVersion>
     <artifactId>neo4j-spatial</artifactId>
     <groupId>org.neo4j</groupId>
-    <version>0.27.2-neo4j-4.2.3</version>
+    <version>0.28.0-neo4j-4.2.3</version>
     <name>Neo4j - Spatial Components</name>
     <description>Spatial utilities and components for Neo4j</description>
     <url>http://components.neo4j.org/${project.artifactId}/${project.version}</url>

src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java

@@ -28,7 +28,6 @@
 import org.neo4j.gis.spatial.rtree.Envelope;
 import org.neo4j.gis.spatial.rtree.Listener;
 import org.neo4j.gis.spatial.rtree.NullListener;
-import org.neo4j.gis.spatial.utilities.ReferenceNodes;
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.schema.IndexDefinition;
 import org.neo4j.graphdb.traversal.Evaluators;
@@ -41,10 +40,13 @@
 import org.neo4j.kernel.impl.traversal.MonoDirectionalTraversalDescription;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
 
+import javax.xml.bind.DatatypeConverter;
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 import java.io.*;
 import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
@@ -392,7 +394,7 @@ private OSMWriter(StatsManager statsManager, OSMImporter osmImporter) {
             this.osmImporter = osmImporter;
         }
 
-        static OSMWriter<WrappedNode> fromGraphDatabase(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager stats, OSMImporter osmImporter, int txInterval) {
+        static OSMWriter<WrappedNode> fromGraphDatabase(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager stats, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException {
             return new OSMGraphWriter(graphDb, securityContext, stats, osmImporter, txInterval);
         }
 
@@ -867,18 +869,29 @@ private static class OSMGraphWriter extends OSMWriter<WrappedNode> {
         private IndexDefinition relationIndex;
         private IndexDefinition changesetIndex;
         private IndexDefinition userIndex;
+        private final String layerHash;
+        private final HashMap<Label, Label> hashedLabels = new HashMap<>();
 
-        private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager statsManager, OSMImporter osmImporter, int txInterval) {
+        private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager statsManager, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException {
             super(statsManager, osmImporter);
             this.graphDb = graphDb;
             this.securityContext = securityContext;
             this.txInterval = txInterval;
             if (this.txInterval < 100) {
                 System.err.println("Warning: Unusually short txInterval, expect bad insert performance");
             }
+            this.layerHash = md5Hash(osmImporter.layerName);
             checkTx(null); // Opens transaction for future writes
         }
 
+        private static String md5Hash(String text) throws NoSuchAlgorithmException {
+            MessageDigest md = MessageDigest.getInstance("MD5");
+            md.update(text.getBytes());
+            byte[] digest = md.digest();
+            String hashed = DatatypeConverter.printHexBinary(digest).toUpperCase();
+            return hashed;
+        }
+
         private void successTx() {
             if (tx != null) {
                 tx.commit();
@@ -930,14 +943,19 @@ private void recoverNode(WrappedNode outOfTx) {
             }
         }
 
-        private WrappedNode findNode(Label label, String name) {
-            Node node = tx.findNode(label, "name", name);
+        private WrappedNode findNodeByName(Label label, String name) {
+            Node node = findNodeByLabelProperty(tx, label, "name", name);
             if (node != null) {
                 return WrappedNode.fromNode(node);
             }
             return null;
         }
 
+        private WrappedNode createNodeWithLabel(Transaction tx, Label label) {
+            Label hashed = getLabelHashed(label);
+            return WrappedNode.fromNode(tx.createNode(label, hashed));
+        }
+
         @Override
         protected void startWays() {
             System.out.println("About to create node index");
@@ -968,12 +986,29 @@ protected void optimize() {
             }
         }
 
+        private Label getLabelHashed(Label label) {
+            if (hashedLabels.containsKey(label)) {
+                return hashedLabels.get(label);
+            } else {
+                Label hashed = Label.label(label.name() + "_" + layerHash);
+                hashedLabels.put(label, hashed);
+                return hashed;
+            }
+        }
+
+        private Node findNodeByLabelProperty(Transaction tx, Label label, String propertyKey, Object value) {
+            Label hashed = getLabelHashed(label);
+            return tx.findNode(hashed, propertyKey, value);
+        }
+
         private IndexDefinition createIndex(Label label, String propertyKey) {
-            IndexDefinition index = findIndex(tx, label, propertyKey);
+            Label hashed = getLabelHashed(label);
+            String indexName = String.format("OSM-%s-%s-%s", osmImporter.layerName, hashed.name(), propertyKey);
+            IndexDefinition index = findIndex(tx, indexName, hashed, propertyKey);
             if (index == null) {
                 successTx();
                 try (Transaction indexTx = beginIndexTx(graphDb)) {
-                    index = indexTx.schema().indexFor(label).on(propertyKey).create();
+                    index = indexTx.schema().indexFor(hashed).on(propertyKey).withName(indexName).create();
                     indexTx.commit();
                 }
                 System.out.println("Created index " + index.getName());
@@ -990,24 +1025,28 @@ private IndexDefinition createIndexIfNotNull(IndexDefinition index, Label label,
             return index;
         }
 
-        private IndexDefinition findIndex(Transaction tx, Label label, String propertyKey) {
+        private IndexDefinition findIndex(Transaction tx, String indexName, Label label, String propertyKey) {
             for (IndexDefinition index : tx.schema().getIndexes(label)) {
                 for (String prop : index.getPropertyKeys()) {
                     if (prop.equals(propertyKey)) {
-                        return index;
+                        if (index.getName().equals(indexName)) {
+                            return index;
+                        } else {
+                            throw new IllegalStateException(String.format("Found pre-existing index '%s' for index '%s'", index.getName(), indexName));
+                        }
                     }
                 }
             }
             return null;
         }
 
         private WrappedNode getOrCreateNode(Label label, String name, String type) {
-            WrappedNode node = findNode(label, name);
+            WrappedNode node = findNodeByName(label, name);
             if (node == null) {
-                Node n = tx.createNode(label);
+                WrappedNode n = createNodeWithLabel(tx, label);
                 n.setProperty("name", name);
                 n.setProperty("type", type);
-                node = checkTx(WrappedNode.fromNode(n));
+                node = checkTx(n);
             }
             return node;
         }
@@ -1038,9 +1077,9 @@ protected void addNodeTags(WrappedNode node, LinkedHashMap<String, Object> tags,
             logNodeAddition(tags, type);
             if (node != null && tags.size() > 0) {
                 statsManager.addToTagStats(type, tags.keySet());
-                Node tagsNode = tx.createNode(LABEL_TAGS);
-                addProperties(tagsNode, tags);
-                node.createRelationshipTo(new WrappedNode(tagsNode), OSMRelation.TAGS);
+                WrappedNode tagsNode = createNodeWithLabel(tx, LABEL_TAGS);
+                addProperties(tagsNode.inner, tags);
+                node.createRelationshipTo(tagsNode, OSMRelation.TAGS);
                 tags.clear();
             }
         }
@@ -1060,12 +1099,12 @@ protected void addNodeGeometry(WrappedNode node, int gtype, Envelope bbox, int v
 
         @Override
         protected WrappedNode addNode(Label label, Map<String, Object> properties, String indexKey) {
-            Node node = tx.createNode(label);
+            WrappedNode node = createNodeWithLabel(tx, label);
             if (indexKey != null && properties.containsKey(indexKey)) {
                 properties.put(indexKey, Long.parseLong(properties.get(indexKey).toString()));
             }
-            addProperties(node, properties);
-            return checkTx(WrappedNode.fromNode(node));
+            addProperties(node.inner, properties);
+            return checkTx(node);
         }
 
         @Override
@@ -1085,7 +1124,7 @@ protected long getDatasetId() {
 
         @Override
         protected WrappedNode getSingleNode(Label label, String property, Object value) {
-            Node node = tx.findNode(LABEL_NODE, property, value);
+            Node node = findNodeByLabelProperty(tx, LABEL_NODE, property, value);
             return node == null ? null : WrappedNode.fromNode(node);
         }
 
@@ -1116,7 +1155,7 @@ protected WrappedNode getOSMNode(long osmId, WrappedNode changesetNode) {
             WrappedNode node = changesetNodes.get(osmId);
             if (node == null) {
                 logNodeFoundFrom("node-index");
-                node = WrappedNode.fromNode(tx.findNode(LABEL_NODE, PROP_NODE_ID, osmId));
+                node = WrappedNode.fromNode(findNodeByLabelProperty(tx, LABEL_NODE, PROP_NODE_ID, osmId));
             } else {
                 logNodeFoundFrom(PROP_CHANGESET);
             }
@@ -1157,7 +1196,7 @@ protected WrappedNode getChangesetNode(Map<String, Object> nodeProps, WrappedNod
                 if (changeset != currentChangesetId) {
                     changesetIndex = createIndexIfNotNull(changesetIndex, LABEL_CHANGESET, PROP_CHANGESET);
                     currentChangesetId = changeset;
-                    Node changesetNode = tx.findNode(LABEL_CHANGESET, PROP_CHANGESET, currentChangesetId);
+                    Node changesetNode = findNodeByLabelProperty(tx, LABEL_CHANGESET, PROP_CHANGESET, currentChangesetId);
                     if (changesetNode != null) {
                         currentChangesetNode = WrappedNode.fromNode(changesetNode);
                     } else {
@@ -1187,7 +1226,7 @@ protected WrappedNode getUserNode(Map<String, Object> nodeProps) {
                 if (uid != currentUserId) {
                     currentUserId = uid;
                     userIndex = createIndexIfNotNull(userIndex, LABEL_USER, PROP_USER_ID);
-                    Node userNode = tx.findNode(LABEL_USER, PROP_USER_ID, currentUserId);
+                    Node userNode = findNodeByLabelProperty(tx, LABEL_USER, PROP_USER_ID, currentUserId);
                     if (userNode != null) {
                         currentUserNode = WrappedNode.fromNode(userNode);
                     } else {
@@ -1198,7 +1237,7 @@ protected WrappedNode getUserNode(Map<String, Object> nodeProps) {
                         currentUserNode = addNode(LABEL_USER, userProps, PROP_USER_ID);
                         userCount++;
                         if (usersNode == null) {
-                            usersNode = WrappedNode.fromNode(tx.createNode(LABEL_USER));
+                            usersNode = createNodeWithLabel(tx, LABEL_USER);
                             osm_dataset.createRelationshipTo(usersNode, OSMRelation.USERS);
                         }
                         usersNode.createRelationshipTo(currentUserNode, OSMRelation.OSM_USER);
@@ -1218,15 +1257,15 @@ public String toString() {
 
     }
 
-    public void importFile(GraphDatabaseService database, String dataset) throws IOException, XMLStreamException {
+    public void importFile(GraphDatabaseService database, String dataset) throws Exception {
         importFile(database, dataset, false, 5000);
     }
 
-    public void importFile(GraphDatabaseService database, String dataset, int txInterval) throws IOException, XMLStreamException {
+    public void importFile(GraphDatabaseService database, String dataset, int txInterval) throws Exception {
         importFile(database, dataset, false, txInterval);
     }
 
-    public void importFile(GraphDatabaseService database, String dataset, boolean allPoints, int txInterval) throws IOException, XMLStreamException {
+    public void importFile(GraphDatabaseService database, String dataset, boolean allPoints, int txInterval) throws Exception {
         importFile(OSMWriter.fromGraphDatabase(database, securityContext, stats, this, txInterval), dataset, allPoints, charset);
     }
 

src/test/java/org/neo4j/gis/spatial/Neo4jSpatialDataStoreTest.java

@@ -35,7 +35,7 @@ public class Neo4jSpatialDataStoreTest {
     public GraphDatabaseService graph;
 
     @Before
-    public void setup() throws IOException, XMLStreamException {
+    public void setup() throws Exception {
         this.databases = new TestDatabaseManagementServiceBuilder(Path.of("target", "test")).impermanent().build();
         this.graph = databases.database(DEFAULT_DATABASE_NAME);
         OSMImporter importer = new OSMImporter("map", new ConsoleListener());

src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java

@@ -929,6 +929,22 @@ public void import_osm_to_layer() {
         testCallCount(db, "CALL spatial.layers()", null, 1);
     }
 
+    @Test
+    public void import_osm_twice_should_pass_with_different_layers() {
+        execute("CALL spatial.addLayer('geom1','OSM','')");
+        execute("CALL spatial.addLayer('geom2','OSM','')");
+
+        testCountQuery("importOSM", "CALL spatial.importOSMToLayer('geom1','map.osm')", 55, "count", null);
+        testCallCount(db, "CALL spatial.layers()", null, 2);
+        testCallCount(db, "CALL spatial.withinDistance('geom1',{lon:6.3740429666,lat:50.93676351666},10000)", null, 217);
+        testCallCount(db, "CALL spatial.withinDistance('geom2',{lon:6.3740429666,lat:50.93676351666},10000)", null, 0);
+
+        testCountQuery("importOSM", "CALL spatial.importOSMToLayer('geom2','map.osm')", 55, "count", null);
+        testCallCount(db, "CALL spatial.layers()", null, 2);
+        testCallCount(db, "CALL spatial.withinDistance('geom1',{lon:6.3740429666,lat:50.93676351666},10000)", null, 217);
+        testCallCount(db, "CALL spatial.withinDistance('geom2',{lon:6.3740429666,lat:50.93676351666},10000)", null, 217);
+    }
+
     @Ignore
     public void import_cracow_to_layer() {
         execute("CALL spatial.addLayer('geom','OSM','')");