Skip to content

Commit 1c904b5

Browse files
Lorenz BuehmannAklakan
Lorenz Buehmann
authored andcommitted
GH-3026: spatial index per graph and kryo serialization
1 parent a3983b2 commit 1c904b5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2624
-154
lines changed
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.jena.system;
19+
20+
import java.util.Objects;
21+
22+
import org.apache.jena.query.ReadWrite;
23+
import org.apache.jena.query.TxnType;
24+
import org.apache.jena.sparql.core.Transactional;
25+
26+
/**
27+
* Txn variant for use with try-with-resources. Allows raising
28+
* checked exceptions in an idiomatic way. Closing the TxnCtl
29+
* instance will by default abort the transaction unless it
30+
* has been manually committed.
31+
* <p>
32+
*
33+
* Usage example:
34+
* <pre>
35+
* public void myMethod() throws IOException {
36+
* try (TxnCtl txn = TxnCtl.begin(dataset, TxnType.WRITE)) {
37+
* // Do work
38+
* // Must manually call commit on success.
39+
* txn.commit();
40+
* }
41+
* }
42+
* </pre>
43+
*
44+
* @param txn The transactional.
45+
* @param txnType The transaction type.
46+
* @return An instance of AutoCloseable that commits the transaction on success.
47+
*/
48+
public class TxnCtl
49+
implements AutoCloseable
50+
{
51+
private Transactional txn;
52+
private boolean b;
53+
54+
private TxnCtl(Transactional txn, boolean b) {
55+
super();
56+
this.txn = txn;
57+
this.b = b;
58+
}
59+
60+
public static TxnCtl begin(Transactional txn, ReadWrite readWrite) {
61+
return begin(txn, TxnType.convert(readWrite));
62+
}
63+
64+
public static TxnCtl begin(Transactional txn, TxnType txnType) {
65+
Objects.requireNonNull(txn);
66+
Objects.requireNonNull(txnType);
67+
boolean b = txn.isInTransaction();
68+
if ( b )
69+
TxnOp.compatibleWithPromote(txnType, txn);
70+
else
71+
txn.begin(txnType);
72+
return new TxnCtl(txn, b);
73+
}
74+
75+
public void commit() {
76+
if ( txn.isInTransaction() ) {
77+
78+
// May have been explicit commit or abort.
79+
txn.commit();
80+
}
81+
}
82+
83+
@Override
84+
public void close() {
85+
if ( !b ) {
86+
if ( txn.isInTransaction() )
87+
// May have been explicit commit or abort.
88+
txn.abort();
89+
txn.end();
90+
}
91+
}
92+
}

jena-fuseki2/jena-fuseki-geosparql/src/main/java/org/apache/jena/fuseki/geosparql/DatasetOperations.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,10 +228,10 @@ private static void prepareSpatialExtension(Dataset dataset, ArgsConfig argsConf
228228
if (!isEmpty) {
229229
if (argsConfig.getSpatialIndexFile() != null) {
230230
File spatialIndexFile = argsConfig.getSpatialIndexFile();
231-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile);
231+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath());
232232
} else if (argsConfig.isTDBFileSetup()) {
233233
File spatialIndexFile = new File(argsConfig.getTdbFile(), SPATIAL_INDEX_FILE);
234-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile);
234+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath());
235235
} else {
236236
GeoSPARQLConfig.setupSpatialIndex(dataset);
237237
}

jena-geosparql/pom.xml

Lines changed: 99 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,16 @@
3434

3535
<dependencies>
3636

37+
<dependency>
38+
<groupId>io.github.galbiston</groupId>
39+
<artifactId>expiring-map</artifactId>
40+
</dependency>
41+
42+
<dependency>
43+
<groupId>javax.xml.bind</groupId>
44+
<artifactId>jaxb-api</artifactId>
45+
</dependency>
46+
3747
<dependency>
3848
<groupId>org.apache.jena</groupId>
3949
<artifactId>jena-arq</artifactId>
@@ -44,18 +54,6 @@
4454
<groupId>org.apache.sis.core</groupId>
4555
<artifactId>sis-referencing</artifactId>
4656
</dependency>
47-
48-
<!-- Evntually this wil be:
49-
<dependency>
50-
<groupId>jakarta.xml.bind</groupId>
51-
<artifactId>jakarta.xml.bind-api</artifactId>
52-
</dependency>
53-
-->
54-
55-
<dependency>
56-
<groupId>javax.xml.bind</groupId>
57-
<artifactId>jaxb-api</artifactId>
58-
</dependency>
5957

6058
<dependency>
6159
<groupId>org.slf4j</groupId>
@@ -82,6 +80,90 @@
8280
<artifactId>commons-collections4</artifactId>
8381
</dependency>
8482

83+
<!-- Kryo serialization via Apache Sedona -->
84+
85+
<!--
86+
<dependency>
87+
<groupId>org.apache.sedona</groupId>
88+
<artifactId>sedona-spark-common-3.5_2.13</artifactId>
89+
<version>1.6.0</version>
90+
<exclusions>
91+
<exclusion>
92+
<groupId>org.apache.spark</groupId>
93+
<artifactId>spark-core_2.12</artifactId>
94+
</exclusion>
95+
<exclusion>
96+
<groupId>org.apache.hadoop</groupId>
97+
<artifactId>hadoop-minicluster</artifactId>
98+
</exclusion>
99+
<exclusion>
100+
<groupId>org.apache.hadoop</groupId>
101+
<artifactId>hadoop-client</artifactId>
102+
</exclusion>
103+
<exclusion>
104+
<groupId>org.geotools</groupId>
105+
<artifactId>gt-geotiff</artifactId>
106+
</exclusion>
107+
<exclusion>
108+
<groupId>org.geotools</groupId>
109+
<artifactId>gt-coverage</artifactId>
110+
</exclusion>
111+
</exclusions>
112+
</dependency>
113+
-->
114+
115+
<dependency>
116+
<groupId>org.apache.sedona</groupId>
117+
<artifactId>sedona-common</artifactId>
118+
<version>1.6.0</version>
119+
<exclusions>
120+
<exclusion>
121+
<groupId>org.apache.spark</groupId>
122+
<artifactId>spark-core_2.12</artifactId>
123+
</exclusion>
124+
<exclusion>
125+
<groupId>org.apache.hadoop</groupId>
126+
<artifactId>hadoop-minicluster</artifactId>
127+
</exclusion>
128+
<exclusion>
129+
<groupId>org.apache.hadoop</groupId>
130+
<artifactId>hadoop-client</artifactId>
131+
</exclusion>
132+
<exclusion>
133+
<groupId>org.geotools</groupId>
134+
<artifactId>gt-geotiff</artifactId>
135+
</exclusion>
136+
<exclusion>
137+
<groupId>org.geotools</groupId>
138+
<artifactId>gt-coverage</artifactId>
139+
</exclusion>
140+
</exclusions>
141+
</dependency>
142+
143+
<!-- H3Utils only exists in newer sedona-common versions -->
144+
<!--
145+
<dependency>
146+
<groupId>org.apache.sedona</groupId>
147+
<artifactId>sedona-common</artifactId>
148+
<version>1.6.0</version>
149+
</dependency>
150+
-->
151+
152+
<dependency>
153+
<groupId>com.esotericsoftware</groupId>
154+
<artifactId>kryo</artifactId>
155+
<version>4.0.2</version>
156+
</dependency>
157+
158+
<!-- Resolve conflicts between jenax/tika/nashorn/hadoop/kryo-->
159+
<!--
160+
<dependency>
161+
<groupId>org.ow2.asm</groupId>
162+
<artifactId>asm</artifactId>
163+
<version>8.0.1</version>
164+
</dependency>
165+
-->
166+
85167
<!-- Non-free; testing only -->
86168
<dependency>
87169
<groupId>org.apache.sis.non-free</groupId>
@@ -102,6 +184,8 @@
102184
<scope>test</scope>
103185
</dependency>
104186

187+
188+
105189
</dependencies>
106190

107191
<build>
@@ -133,11 +217,11 @@
133217
<plugin>
134218
<groupId>org.apache.maven.plugins</groupId>
135219
<artifactId>maven-source-plugin</artifactId>
136-
<executions>
220+
<executions>
137221
<execution>
138-
<id>attach-sources-test</id>
222+
<id>attach-sources-test</id>
139223
<goals>
140-
<goal>test-jar-no-fork</goal>
224+
<goal>test-jar-no-fork</goal>
141225
</goals>
142226
</execution>
143227
</executions>

jena-geosparql/src/main/java/org/apache/jena/geosparql/InitGeoSPARQL.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public static void init() {
4242
if ( initialized )
4343
return ;
4444
synchronized (initLock) {
45-
if ( initialized ) {
45+
if ( initialized || System.getProperty("jena.geosparql.skip", "false").equalsIgnoreCase("true") ) {
4646
JenaSystem.logLifecycle("InitGeoSPARQL - skip") ;
4747
return ;
4848
}

jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/GeoAssembler.java

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,16 @@
1818

1919
package org.apache.jena.geosparql.assembler;
2020

21-
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.*;
21+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pApplyDefaultGeometry;
22+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pDataset;
23+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexEnabled;
24+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexExpiries;
25+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexSizes;
26+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pInference;
27+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pQueryRewrite;
28+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSpatialIndexFile;
29+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSpatialIndexPerGraph;
30+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSrsUri;
2231
import static org.apache.jena.sparql.util.graph.GraphUtils.getBooleanValue;
2332

2433
import java.io.IOException;
@@ -34,6 +43,7 @@
3443
import org.apache.jena.geosparql.configuration.GeoSPARQLOperations;
3544
import org.apache.jena.geosparql.configuration.SrsException;
3645
import org.apache.jena.geosparql.spatial.SpatialIndexException;
46+
import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexUtils;
3747
import org.apache.jena.graph.Graph;
3848
import org.apache.jena.graph.Node;
3949
import org.apache.jena.query.Dataset;
@@ -117,9 +127,21 @@ public DatasetGraph createDataset(Assembler a, Resource root) {
117127
if (root.hasProperty(pSpatialIndexFile) )
118128
spatialIndexFilename = GraphUtils.getStringValue(root, pSpatialIndexFile);
119129

120-
// ---- Build
130+
// spatial index per named graph option
131+
boolean spatialIndexPerGraph = false;
132+
if (root.hasProperty(pSpatialIndexPerGraph) )
133+
spatialIndexPerGraph = getBooleanValue(root, pSpatialIndexPerGraph);
134+
135+
// SRS URI
136+
String srsURI = null;
137+
if (root.hasProperty(pSrsUri) )
138+
srsURI = GraphUtils.getStringValue(root, pSrsUri);
121139

140+
141+
// ---- Build
122142
Dataset dataset = DatasetFactory.wrap(base);
143+
dataset.getContext().set(SpatialIndexUtils.symSpatialIndexPerGraph, spatialIndexPerGraph);
144+
dataset.getContext().set(SpatialIndexUtils.symSrsUri, srsURI);
123145

124146
// Conversion of data. Startup-only.
125147
// needed for w3c:geo/wgs84_pos#lat/log.
@@ -142,13 +164,13 @@ public DatasetGraph createDataset(Assembler a, Resource root) {
142164
//Setup GeoSPARQL
143165
if (indexEnabled) {
144166
GeoSPARQLConfig.setupMemoryIndex(indexSizes.get(0), indexSizes.get(1), indexSizes.get(2),
145-
(long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2),
146-
queryRewrite);
167+
(long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2),
168+
queryRewrite);
147169
} else {
148170
GeoSPARQLConfig.setupNoIndex(queryRewrite);
149171
}
150172

151-
prepareSpatialExtension(dataset, spatialIndexFilename);
173+
prepareSpatialExtension(dataset, spatialIndexFilename, spatialIndexPerGraph);
152174
return base;
153175
}
154176

@@ -165,8 +187,8 @@ private static List<Integer> getListInteger(Resource r, Property p, int len) {
165187
return integerList;
166188
}
167189

168-
private static void prepareSpatialExtension(Dataset dataset, String spatialIndex){
169-
boolean isEmpty = dataset.calculateRead(()->dataset.isEmpty());
190+
private static void prepareSpatialExtension(Dataset dataset, String spatialIndex, boolean spatialIndexPerGraph){
191+
boolean isEmpty = dataset.calculateRead(dataset::isEmpty);
170192
if ( isEmpty && spatialIndex != null ) {
171193
LOG.warn("Dataset empty. Spatial Index not constructed. Server will require restarting after adding data and any updates to build Spatial Index.");
172194
return;
@@ -185,12 +207,12 @@ private static void prepareSpatialExtension(Dataset dataset, String spatialIndex
185207
// file given but empty -> compute and serialize index
186208
Path spatialIndexPath = Path.of(spatialIndex);
187209
if ( ! Files.exists(spatialIndexPath) || Files.size(spatialIndexPath) == 0 ) {
188-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath.toFile());
210+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath, spatialIndexPerGraph);
189211
return;
190212
}
191213

192214
// load and setup the precomputed index
193-
GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath.toFile());
215+
GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath);
194216
}
195217
catch (SrsException ex) {
196218
// Data but no spatial data.

jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/VocabGeoSPARQL.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ private static Property property(String shortName) {
7373
// "File to load or store the spatial index. Default to " + SPATIAL_INDEX_FILE + " in TDB folder if using TDB and not set. Otherwise spatial index is not stored.
7474
public static final Property pSpatialIndexFile = property("spatialIndexFile");
7575

76+
public static final Property pSpatialIndexPerGraph = property("spatialIndexPerGraph");
77+
78+
public static final Property pSrsUri = property("srsUri");
79+
7680
// Dataset
7781
public static final Property pDataset = property("dataset");
7882
}

0 commit comments

Comments
 (0)