Skip to content

Commit 84417b9

Browse files
craigtavernercraigtaverner
committed
Port to Neo4j 4.1.7
Most some internal API changes: * Transaction type names * IndexSeek API change * Stricter checks on parameters containing Node objects This last change means that if we return a Node from one transaction, we cannot pass it into a spatial procedure in another transaction. Users have to either lookup the node again in the new transaction, or use some new procedures we provide here: * spatial.addNode.byId * spatial.addNodes.byId * spatial.removeNode.byId * spatial.removeNodes.byId Also the removeNode procedures now return `nodeId` instead of `node`
1 parent 94a4ed7 commit 84417b9

File tree

9 files changed

+105
-46
lines changed

9 files changed

+105
-46
lines changed

‎README.md

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,14 @@ This has meant that the spatial library needed a major refactoring to work with
6464
It was therefor necessary to upgrade the GeoTools libraries to version 24.2.
6565
This in turn required a re-write of the Neo4jDataStore interface since the older API had
6666
long been deprecated, and was entirely unavailable in newer versions.
67+
* Neo4j 4.1 was slightly stricter with regards to passing nodes as parameters, requiring the nodes
68+
objects to have been created in the current transaction.
69+
To work around this we added `.byId` versions of the `spatial.addNode` and `spatial.removeNode` procedures.
70+
We also changed the `spatial.removeNode` procedures to return `nodeId` instead of `node`.
6771

68-
Consequences of this port:
72+
Consequences of the port to Neo4j 4.x:
6973

70-
* The large number of changes mean that the 0.27.0 version should be considered very alpha.
74+
* The large number of changes mean that the 0.27.x versions should be considered very alpha.
7175
* Many API's have changed and client code might need to be adapted to take the changes into account.
7276
* The new DataStore API is entirely untested in GeoServer, besides the existing unit and integration tests.
7377
* The need to manage threads and create schema indexes results in the procedures requiring
@@ -335,6 +339,7 @@ The Neo4j Spatial Plugin is available for inclusion in the server version of Neo
335339
* [v0.26.2 for Neo4j 3.5.2](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.26.2-neo4j-3.5.2/neo4j-spatial-0.26.2-neo4j-3.5.2-server-plugin.jar?raw=true)
336340
* Using GeoTools 24.2 (for GeoServer 2.18.x):
337341
* [v0.27.0 for Neo4j 4.0.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.0-neo4j-4.0.3/neo4j-spatial-0.27.0-neo4j-4.0.3-server-plugin.jar?raw=true)
342+
* [v0.27.1 for Neo4j 4.1.7](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.1-neo4j-4.1.7/neo4j-spatial-0.27.1-neo4j-4.1.7-server-plugin.jar?raw=true)
338343

339344
For versions up to 0.15-neo4j-2.3.4:
340345

@@ -451,7 +456,7 @@ Add the following repositories and dependency to your project's pom.xml:
451456
<dependency>
452457
<groupId>org.neo4j</groupId>
453458
<artifactId>neo4j-spatial</artifactId>
454-
<version>0.27.0-neo4j-4.0.3</version>
459+
<version>0.27.1-neo4j-4.1.7</version>
455460
</dependency>
456461
~~~
457462

‎pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
33
<properties>
4-
<neo4j.version>4.0.3</neo4j.version>
4+
<neo4j.version>4.1.7</neo4j.version>
55
<lucene.version>8.2.0</lucene.version>
66
<!-- make sure lucene version is the same as the one the current neo4j depends on -->
77
<neo4j.java.version>11</neo4j.java.version>
@@ -23,7 +23,7 @@
2323
<modelVersion>4.0.0</modelVersion>
2424
<artifactId>neo4j-spatial</artifactId>
2525
<groupId>org.neo4j</groupId>
26-
<version>0.27.0-neo4j-4.0.3</version>
26+
<version>0.27.1-neo4j-4.1.7</version>
2727
<name>Neo4j - Spatial Components</name>
2828
<description>Spatial utilities and components for Neo4j</description>
2929
<url>http://components.neo4j.org/${project.artifactId}/${project.version}</url>

‎src/main/java/org/neo4j/gis/spatial/AbstractGeometryEncoder.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*/
2020
package org.neo4j.gis.spatial;
2121

22-
import org.apache.commons.lang.ArrayUtils;
22+
import org.apache.commons.lang3.ArrayUtils;
2323
import org.neo4j.gis.spatial.rtree.Envelope;
2424
import org.neo4j.graphdb.Node;
2525
import org.neo4j.graphdb.Entity;

‎src/main/java/org/neo4j/gis/spatial/index/IndexManager.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,14 +150,14 @@ private IndexMaker(String indexName, Label label, String propertyKey) {
150150
@Override
151151
public void run() {
152152
try {
153-
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.explicit, securityContext)) {
153+
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.EXPLICIT, securityContext)) {
154154
index = findIndex(tx);
155155
if (index == null) {
156156
index = tx.schema().indexFor(label).withName(indexName).on(propertyKey).create();
157157
}
158158
tx.commit();
159159
}
160-
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.explicit, securityContext)) {
160+
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.EXPLICIT, securityContext)) {
161161
tx.schema().awaitIndexOnline(indexName, 30, TimeUnit.SECONDS);
162162
}
163163
} catch (Exception e) {
@@ -208,7 +208,7 @@ private IndexRemover(IndexDefinition index) {
208208
@Override
209209
public void run() {
210210
try {
211-
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.explicit, securityContext)) {
211+
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.EXPLICIT, securityContext)) {
212212
// Need to find and drop in the same transaction due to saved state in the index definition implementation
213213
IndexDefinition found = tx.schema().getIndexByName(index.getName());
214214
if (found != null) {

‎src/main/java/org/neo4j/gis/spatial/index/LayerSpaceFillingCurvePointIndex.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,16 @@
2626
import org.neo4j.gis.spatial.index.curves.StandardConfiguration;
2727
import org.neo4j.gis.spatial.rtree.filter.AbstractSearchEnvelopeIntersection;
2828
import org.neo4j.gis.spatial.rtree.filter.SearchFilter;
29-
import org.neo4j.graphdb.*;
29+
import org.neo4j.graphdb.Label;
30+
import org.neo4j.graphdb.Node;
31+
import org.neo4j.graphdb.ResourceIterator;
32+
import org.neo4j.graphdb.Transaction;
3033
import org.neo4j.internal.helpers.collection.Iterators;
3134
import org.neo4j.internal.kernel.api.*;
3235
import org.neo4j.internal.schema.IndexDescriptor;
33-
import org.neo4j.internal.schema.IndexOrder;
3436
import org.neo4j.internal.schema.IndexType;
3537
import org.neo4j.internal.schema.SchemaDescriptor;
38+
import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer;
3639
import org.neo4j.kernel.api.KernelTransaction;
3740
import org.neo4j.kernel.impl.core.NodeEntity;
3841
import org.neo4j.kernel.impl.coreapi.internal.NodeCursorResourceIterator;
@@ -142,9 +145,9 @@ private ResourceIterator<Node> nodesByLabelAndProperty(KernelTransaction transac
142145
// Ha! We found an index - let's use it to find matching nodes
143146
try
144147
{
145-
NodeValueIndexCursor cursor = transaction.cursors().allocateNodeValueIndexCursor();
148+
NodeValueIndexCursor cursor = transaction.cursors().allocateNodeValueIndexCursor(PageCursorTracer.NULL);
146149
IndexReadSession indexSession = read.indexReadSession( index );
147-
read.nodeIndexSeek( indexSession, cursor, IndexOrder.NONE, false, query );
150+
read.nodeIndexSeek( indexSession, cursor, IndexQueryConstraints.unordered(false), query );
148151

149152
return new NodeCursorResourceIterator<>( cursor, (id) -> new NodeEntity(transaction.internalTransaction(), id) );
150153
}

‎src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,11 @@
3333
import org.neo4j.graphdb.schema.IndexDefinition;
3434
import org.neo4j.graphdb.traversal.Evaluators;
3535
import org.neo4j.graphdb.traversal.TraversalDescription;
36-
import org.neo4j.internal.kernel.api.security.AccessMode;
3736
import org.neo4j.internal.kernel.api.security.SecurityContext;
3837
import org.neo4j.io.fs.FileUtils;
3938
import org.neo4j.io.layout.DatabaseLayout;
4039
import org.neo4j.io.layout.Neo4jLayout;
4140
import org.neo4j.kernel.api.KernelTransaction;
42-
import org.neo4j.kernel.impl.api.security.OverriddenAccessMode;
4341
import org.neo4j.kernel.impl.traversal.MonoDirectionalTraversalDescription;
4442
import org.neo4j.kernel.internal.GraphDatabaseAPI;
4543

@@ -202,7 +200,7 @@ private Transaction beginTx(GraphDatabaseService database) {
202200
if (!(database instanceof GraphDatabaseAPI)) {
203201
throw new IllegalArgumentException("database must implement GraphDatabaseAPI");
204202
}
205-
return ((GraphDatabaseAPI) database).beginTransaction(KernelTransaction.Type.explicit, securityContext);
203+
return ((GraphDatabaseAPI) database).beginTransaction(KernelTransaction.Type.EXPLICIT, securityContext);
206204
}
207205

208206
public long reIndex(GraphDatabaseService database) {
@@ -902,7 +900,7 @@ private static Transaction beginTx(GraphDatabaseService database, SecurityContex
902900
if (!(database instanceof GraphDatabaseAPI)) {
903901
throw new IllegalArgumentException("database must implement GraphDatabaseAPI");
904902
}
905-
return ((GraphDatabaseAPI) database).beginTransaction(KernelTransaction.Type.explicit, securityContext);
903+
return ((GraphDatabaseAPI) database).beginTransaction(KernelTransaction.Type.EXPLICIT, securityContext);
906904
}
907905

908906
private void beginTx() {

‎src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import java.util.*;
6565
import java.util.function.BiFunction;
6666
import java.util.function.Function;
67+
import java.util.stream.Collectors;
6768
import java.util.stream.Stream;
6869

6970
import static org.neo4j.gis.spatial.SpatialDatabaseService.RTREE_INDEX_NAME;
@@ -106,6 +107,14 @@ public NodeResult(Node node) {
106107
}
107108
}
108109

110+
public static class NodeIdResult {
111+
public final long nodeId;
112+
113+
public NodeIdResult(long nodeId) {
114+
this.nodeId = nodeId;
115+
}
116+
}
117+
109118
public static class CountResult {
110119
public final long count;
111120

@@ -490,6 +499,10 @@ private Stream<NodeResult> streamNode(Node node) {
490499
return Stream.of(new NodeResult(node));
491500
}
492501

502+
private Stream<NodeIdResult> streamNode(long nodeId) {
503+
return Stream.of(new NodeIdResult(nodeId));
504+
}
505+
493506
@Procedure(value="spatial.addWKTLayer", mode=WRITE)
494507
@Description("Adds a new WKT layer with the given node property to hold the WKT string, returns the layer root node")
495508
public Stream<NodeResult> addWKTLayer(@Name("name") String name,
@@ -540,12 +553,27 @@ public Stream<CountResult> addNodesToLayer(@Name("layerName") String name, @Name
540553
return Stream.of(new CountResult(layer.addAll(tx, nodes)));
541554
}
542555

556+
@Procedure(value="spatial.addNode.byId", mode=WRITE)
557+
@Description("Adds the given node to the layer, returns the geometry-node")
558+
public Stream<NodeResult> addNodeIdToLayer(@Name("layerName") String name, @Name("nodeId") long nodeId) {
559+
EditableLayer layer = getEditableLayerOrThrow(tx, spatial(), name);
560+
return streamNode(layer.add(tx, tx.getNodeById(nodeId)).getGeomNode());
561+
}
562+
563+
@Procedure(value="spatial.addNodes.byId", mode=WRITE)
564+
@Description("Adds the given nodes list to the layer, returns the count")
565+
public Stream<CountResult> addNodeIdsToLayer(@Name("layerName") String name, @Name("nodeIds") List<Long> nodeIds) {
566+
EditableLayer layer = getEditableLayerOrThrow(tx, spatial(), name);
567+
List<Node> nodes = nodeIds.stream().map(id -> tx.getNodeById(id)).collect(Collectors.toList());
568+
return Stream.of(new CountResult(layer.addAll(tx, nodes)));
569+
}
570+
543571
@Procedure(value="spatial.removeNode", mode=WRITE)
544572
@Description("Removes the given node from the layer, returns the geometry-node")
545-
public Stream<NodeResult> removeNodeFromLayer(@Name("layerName") String name, @Name("node") Node node) {
573+
public Stream<NodeIdResult> removeNodeFromLayer(@Name("layerName") String name, @Name("node") Node node) {
546574
EditableLayer layer = getEditableLayerOrThrow(tx, spatial(), name);
547575
layer.removeFromIndex(tx, node.getId());
548-
return streamNode(node);
576+
return streamNode(node.getId());
549577
}
550578

551579
@Procedure(value="spatial.removeNodes", mode=WRITE)
@@ -561,6 +589,27 @@ public Stream<CountResult> removeNodesFromLayer(@Name("layerName") String name,
561589
return Stream.of(new CountResult(before - after));
562590
}
563591

592+
@Procedure(value="spatial.removeNode.byId", mode=WRITE)
593+
@Description("Removes the given node from the layer, returns the geometry-node")
594+
public Stream<NodeIdResult> removeNodeFromLayer(@Name("layerName") String name, @Name("nodeId") long nodeId) {
595+
EditableLayer layer = getEditableLayerOrThrow(tx, spatial(), name);
596+
layer.removeFromIndex(tx, nodeId);
597+
return streamNode(nodeId);
598+
}
599+
600+
@Procedure(value="spatial.removeNodes.byId", mode=WRITE)
601+
@Description("Removes the given nodes from the layer, returns the count of nodes removed")
602+
public Stream<CountResult> removeNodeIdsFromLayer(@Name("layerName") String name, @Name("nodeIds") List<Long> nodeIds) {
603+
EditableLayer layer = getEditableLayerOrThrow(tx, spatial(), name);
604+
//TODO optimize bulk node removal from RTree like we have done for node additions
605+
int before = layer.getIndex().count(tx);
606+
for (long nodeId : nodeIds) {
607+
layer.removeFromIndex(tx, nodeId);
608+
}
609+
int after = layer.getIndex().count(tx);
610+
return Stream.of(new CountResult(before - after));
611+
}
612+
564613
@Procedure(value="spatial.addWKT", mode=WRITE)
565614
@Description("Adds the given WKT string to the layer, returns the created geometry node")
566615
public Stream<NodeResult> addGeometryWKTToLayer(@Name("layerName") String name, @Name("geometry") String geometryWKT) throws ParseException {
@@ -682,7 +731,7 @@ long getResult() {
682731
public void run() {
683732
// Create the layer in the same thread as doing the import, otherwise we have an outer thread doing a create,
684733
// and the inner thread repeating it, resulting in duplicates
685-
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.explicit, securityContext)) {
734+
try (Transaction tx = db.beginTransaction(KernelTransaction.Type.EXPLICIT, securityContext)) {
686735
layerMaker.apply(tx, layerName);
687736
tx.commit();
688737
}

‎src/test/java/org/neo4j/gis/spatial/Neo4jTestCase.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ public abstract class Neo4jTestCase {
5656
//NORMAL_CONFIG.put( GraphDatabaseSettings.strings_mapped_memory_size.name(), "200M" );
5757
//NORMAL_CONFIG.put( GraphDatabaseSettings.arrays_mapped_memory_size.name(), "0M" );
5858
NORMAL_CONFIG.put(GraphDatabaseSettings.pagecache_memory.name(), "200M");
59-
NORMAL_CONFIG.put(GraphDatabaseSettings.batch_inserter_batch_size.name(), "2");
60-
NORMAL_CONFIG.put(GraphDatabaseSettings.dump_configuration.name(), "false");
6159
}
6260

6361
static final Map<String, String> LARGE_CONFIG = new HashMap<>();
@@ -69,8 +67,6 @@ public abstract class Neo4jTestCase {
6967
//LARGE_CONFIG.put( GraphDatabaseSettings.strings_mapped_memory_size.name(), "800M" );
7068
//LARGE_CONFIG.put( GraphDatabaseSettings.arrays_mapped_memory_size.name(), "10M" );
7169
LARGE_CONFIG.put(GraphDatabaseSettings.pagecache_memory.name(), "100M");
72-
LARGE_CONFIG.put(GraphDatabaseSettings.batch_inserter_batch_size.name(), "2");
73-
LARGE_CONFIG.put(GraphDatabaseSettings.dump_configuration.name(), "true");
7470
}
7571

7672
private static final File basePath = new File("target/var");

‎src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ public static void registerProceduresAndFunctions(GraphDatabaseService db, Class
132132
public void add_node_to_non_existing_layer() {
133133
execute("CALL spatial.addPointLayer('some_name')");
134134
Node node = createNode("CREATE (n:Point {latitude:60.1,longitude:15.2}) RETURN n", "n");
135-
testCallFails(db, "CALL spatial.addNode('wrong_name',$node)", map("node", node), "No such layer 'wrong_name'");
135+
testCallFails(db, "CALL spatial.addNode.byId('wrong_name',$nodeId)", map("nodeId", node.getId()), "No such layer 'wrong_name'");
136136
}
137137

138138
@Test
@@ -694,38 +694,46 @@ public void add_a_node_to_the_spatial_index_short_with_geohash() {
694694
@Test
695695
public void add_two_nodes_to_the_spatial_layer() {
696696
execute("CALL spatial.addPointLayerXY('geom','lon','lat')");
697-
Node node1;
698-
Node node2;
697+
long node1;
698+
long node2;
699699
try (Transaction tx = db.beginTx()) {
700700
Result result = tx.execute("CREATE (n1:Node {lat:60.1,lon:15.2}),(n2:Node {lat:60.1,lon:15.3}) WITH n1,n2 CALL spatial.addNodes('geom',[n1,n2]) YIELD count RETURN n1,n2,count");
701701
Map<String, Object> row = result.next();
702-
node1 = (Node) row.get("n1");
703-
node2 = (Node) row.get("n2");
702+
node1 = ((Node) row.get("n1")).getId();
703+
node2 = ((Node) row.get("n2")).getId();
704704
long count = (Long) row.get("count");
705705
Assert.assertEquals(2L, count);
706706
result.close();
707707
tx.commit();
708708
}
709709
testResult(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", res -> {
710-
assertTrue(res.hasNext());
711-
assertEquals(node1, res.next().get("node"));
712-
assertTrue(res.hasNext());
713-
assertEquals(node2, res.next().get("node"));
714-
assertFalse(res.hasNext());
715-
}
716-
);
710+
assertTrue(res.hasNext());
711+
assertEquals(node1, ((Node) res.next().get("node")).getId());
712+
assertTrue(res.hasNext());
713+
assertEquals(node2, ((Node) res.next().get("node")).getId());
714+
assertFalse(res.hasNext());
715+
});
717716
try (Transaction tx = db.beginTx()) {
718-
Result removeResult = tx.execute("CALL spatial.removeNode('geom',$node) YIELD node RETURN node", map("node", node1));
719-
Assert.assertEquals(node1, removeResult.next().get("node"));
717+
Node node = (Node) tx.execute("MATCH (node) WHERE id(node) = $nodeId RETURN node", map("nodeId", node1)).next().get("node");
718+
Result removeResult = tx.execute("CALL spatial.removeNode('geom',$node) YIELD nodeId RETURN nodeId", map("node", node));
719+
Assert.assertEquals(node1, removeResult.next().get("nodeId"));
720720
removeResult.close();
721721
tx.commit();
722722
}
723723
testResult(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", res -> {
724-
assertTrue(res.hasNext());
725-
assertEquals(node2, res.next().get("node"));
726-
assertFalse(res.hasNext());
727-
}
728-
);
724+
assertTrue(res.hasNext());
725+
assertEquals(node2, ((Node) res.next().get("node")).getId());
726+
assertFalse(res.hasNext());
727+
});
728+
try (Transaction tx = db.beginTx()) {
729+
Result removeResult = tx.execute("CALL spatial.removeNode.byId('geom',$nodeId) YIELD nodeId RETURN nodeId", map("nodeId", node2));
730+
Assert.assertEquals(node2, removeResult.next().get("nodeId"));
731+
removeResult.close();
732+
tx.commit();
733+
}
734+
testResult(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", res -> {
735+
assertFalse(res.hasNext());
736+
});
729737
}
730738

731739
@Test
@@ -793,9 +801,9 @@ private void testRemoveNode(String layer, int count) {
793801
String remove = "UNWIND range(1,$count) as i\n" +
794802
"MATCH (n:Point {id:i})\n" +
795803
"WITH n\n" +
796-
"CALL spatial.removeNode('" + layer + "',n) YIELD node\n" +
797-
"RETURN count(node)";
798-
testCountQuery("removeNode", remove, count / 2, "count(node)", map("count", count / 2));
804+
"CALL spatial.removeNode('" + layer + "',n) YIELD nodeId\n" +
805+
"RETURN count(nodeId)";
806+
testCountQuery("removeNode", remove, count / 2, "count(nodeId)", map("count", count / 2));
799807
// Check that only half remain
800808
testCountQuery("withinDistance", "CALL spatial.withinDistance('" + layer + "',{lon:15.0,lat:60.0},1000) YIELD node RETURN count(node)", count / 2, "count(node)", null);
801809
}

0 commit comments

Comments
 (0)