diff --git a/jena-arq/src/main/java/org/apache/jena/sparql/util/Context.java b/jena-arq/src/main/java/org/apache/jena/sparql/util/Context.java index 63fd6e8da0f..fd2a5f261e3 100644 --- a/jena-arq/src/main/java/org/apache/jena/sparql/util/Context.java +++ b/jena-arq/src/main/java/org/apache/jena/sparql/util/Context.java @@ -18,10 +18,12 @@ package org.apache.jena.sparql.util; -import java.util.*; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; +import java.util.function.BiFunction; import org.apache.jena.atlas.lib.Lib; import org.apache.jena.atlas.logging.Log; @@ -387,6 +389,12 @@ public void clear() { context.clear(); } + /** Atomic compute. */ + @SuppressWarnings("unchecked") + public V compute(Symbol key, BiFunction remappingFunction) { + return (V)context.compute(key, remappingFunction); + } + @Override public String toString() { String x = ""; diff --git a/jena-arq/src/main/java/org/apache/jena/system/AutoTxn.java b/jena-arq/src/main/java/org/apache/jena/system/AutoTxn.java new file mode 100644 index 00000000000..15ff33ee4b8 --- /dev/null +++ b/jena-arq/src/main/java/org/apache/jena/system/AutoTxn.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.system; + +import org.apache.jena.sparql.core.Transactional; + +/** + * Transaction wrapper object for use with try-with-resources. + * See {@linkplain Txn#autoTxn(Transactional, org.apache.jena.query.TxnType)}. + */ +public class AutoTxn + implements AutoCloseable +{ + private Transactional txn; + private boolean isTxnStartedHere; + + /** This constructor is called from {@linkplain Txn#autoTxn(Transactional, org.apache.jena.query.TxnType)}. */ + AutoTxn(Transactional txn, boolean isTxnStartedHere) { + super(); + this.txn = txn; + this.isTxnStartedHere = isTxnStartedHere; + } + + public void commit() { + if ( txn.isInTransaction() ) { + // May have been explicit commit or abort. + txn.commit(); + } + } + + @Override + public void close() { + if ( isTxnStartedHere ) { + if ( txn.isInTransaction() ) + // May have been explicit commit or abort. + txn.abort(); + txn.end(); + } + } +} diff --git a/jena-arq/src/main/java/org/apache/jena/system/Txn.java b/jena-arq/src/main/java/org/apache/jena/system/Txn.java index 603f0fe1451..ff51ab85c25 100644 --- a/jena-arq/src/main/java/org/apache/jena/system/Txn.java +++ b/jena-arq/src/main/java/org/apache/jena/system/Txn.java @@ -18,8 +18,10 @@ package org.apache.jena.system; +import java.util.Objects; import java.util.function.Supplier; +import org.apache.jena.query.ReadWrite; import org.apache.jena.query.TxnType; import org.apache.jena.sparql.core.Transactional; @@ -137,4 +139,46 @@ private static void onThrowable(Throwable th, T txn) { txn.end(); } catch (Throwable th2) { th.addSuppressed(th2); } } + + /** + * Begins a transaction and returns a transaction control instance suitable + * for use with try-with-resources blocks. + * See {@link #autoTxn(Transactional, TxnType)}. + */ + public static AutoTxn autoTxn(Transactional txn, ReadWrite readWrite) { + return autoTxn(txn, TxnType.convert(readWrite)); + } + + /** + * Begins a transaction and returns a {@linkplain AutoTxn} instance suitable + * for use with try-with-resources blocks. + * This allows for raising checked exceptions in an idiomatic way. + * Closing the AutoTxn instance will abort the transaction unless there has + * been an explicit call to {@linkplain AutoTxn#commit()}. + *

+ * + * Usage example: + *

+     * public void myMethod() throws IOException {
+     *   try (AutoTxn txn = Txn.autoTxn(dataset, TxnType.WRITE)) {
+     *     // Do work.
+     *     if (someError) {
+     *         throw new IOException();
+     *     }
+     *     // Explicitly call commit on success.
+     *     txn.commit();
+     *   }
+     * }
+     * 
+ */ + public static AutoTxn autoTxn(Transactional txn, TxnType txnType) { + Objects.requireNonNull(txn); + Objects.requireNonNull(txnType); + boolean isTxnStartedHere = !txn.isInTransaction(); + if ( !isTxnStartedHere ) + TxnOp.compatibleWithPromote(txnType, txn); + else + txn.begin(txnType); + return new AutoTxn(txn, isTxnStartedHere); + } } diff --git a/jena-arq/src/main/java/org/apache/jena/web/HttpSC.java b/jena-arq/src/main/java/org/apache/jena/web/HttpSC.java index cf0607dfe68..2d9a69a527a 100644 --- a/jena-arq/src/main/java/org/apache/jena/web/HttpSC.java +++ b/jena-arq/src/main/java/org/apache/jena/web/HttpSC.java @@ -306,7 +306,7 @@ public boolean isServerError() */ public static Code getCode(int code) { - if (code <= MAX_CODE) + if (code >= 0 && code <= MAX_CODE) { return codeMap[code]; } diff --git a/jena-arq/src/test/java/org/apache/jena/system/TestTxn.java b/jena-arq/src/test/java/org/apache/jena/system/TestTxn.java index f098f95c074..7f174058cdc 100644 --- a/jena-arq/src/test/java/org/apache/jena/system/TestTxn.java +++ b/jena-arq/src/test/java/org/apache/jena/system/TestTxn.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import org.apache.jena.query.ReadWrite; import org.apache.jena.query.TxnType; @@ -31,7 +32,7 @@ public class TestTxn { - TxnCounter counter = new TxnCounter(0) ; + TxnCounter counter = new TxnCounter(0) ; @Test public void txn_basic_01() { long v1 = counter.get() ; @@ -42,7 +43,7 @@ public class TestTxn { } @Test public void txn_basic_02() { - long x = + long x = Txn.calculateRead(counter, () -> { assertEquals("In R, value()", 0, counter.value()) ; assertEquals("In R, get()", 0, counter.get()) ; @@ -53,7 +54,7 @@ public class TestTxn { @Test public void txn_basic_03() { Txn.executeWrite(counter, counter::inc) ; - long x = + long x = Txn.calculateRead(counter, () -> { assertEquals("In R, value()", 1, counter.value()) ; assertEquals("In R, get()", 1, counter.get()) ; @@ -63,7 +64,7 @@ public class TestTxn { } @Test public void txn_basic_05() { - long x = + long x = Txn.calculateWrite(counter, () -> { counter.inc() ; assertEquals("In W, value()", 0, counter.value()) ; @@ -74,7 +75,7 @@ public class TestTxn { } @Test public void txn_write_01() { - long x = + long x = Txn.calculateWrite(counter, () -> { counter.inc() ; assertEquals("In W, value()", 0, counter.value()) ; @@ -87,7 +88,7 @@ public class TestTxn { } @Test public void txn_write_02() { - long x = + long x = Txn.calculateWrite(counter, () -> { counter.inc() ; assertEquals("In W, value()", 0, counter.value()) ; @@ -121,13 +122,13 @@ public class TestTxn { @Test public void txn_rw_1() { assertEquals(0, counter.get()) ; - + Txn.executeWrite(counter, () -> { counter.inc() ; assertEquals("In W, value()", 0, counter.value()) ; assertEquals("In W, get()",1, counter.get()) ; }) ; - + assertEquals("Direct value()", 1, counter.value()) ; assertEquals("Direct get()", 1, counter.get()) ; @@ -148,7 +149,7 @@ public class TestTxn { assertEquals("In W, value()", 0, counter.value()) ; assertEquals("In W, get()",1, counter.get()) ; }) ; - + assertEquals("Direct value()", 1, counter.get()) ; assertEquals("Direct get()", 1, counter.get()) ; @@ -160,11 +161,11 @@ public class TestTxn { @Test public void txn_continue_1() { Txn.executeWrite(counter, ()->counter.set(91)) ; - + Txn.executeWrite(counter, ()-> { assertEquals("In txn, value()", 91, counter.value()) ; assertEquals("In txn, read()", 91, counter.read()) ; - counter.inc(); + counter.inc(); Txn.executeWrite(counter, ()->{ assertEquals("In txn, value()", 91, counter.value()) ; assertEquals("In txn, get()", 92, counter.read()) ; @@ -175,11 +176,11 @@ public class TestTxn { @Test public void txn_continue_2() { Txn.executeWrite(counter, ()->counter.set(91)) ; - + Txn.executeWrite(counter, ()-> { assertEquals("In txn, value()", 91, counter.value()) ; assertEquals("In txn, read()", 91, counter.read()) ; - counter.inc(); + counter.inc(); Txn.executeWrite(counter, ()->{ assertEquals("In txn, value()", 91, counter.value()) ; assertEquals("In txn, get()", 92, counter.read()) ; @@ -195,7 +196,7 @@ public class TestTxn { @Test(expected=ExceptionFromTest.class) public void txn_exception_01() { Txn.executeWrite(counter, counter::inc) ; - + Txn.executeWrite(counter, () -> { counter.inc() ; assertEquals("In W, value()", 1, counter.value()) ; @@ -207,7 +208,7 @@ public void txn_exception_01() { @Test public void txn_exception_02() { Txn.executeWrite(counter, ()->counter.set(8)) ; - + try { Txn.executeWrite(counter, () -> { counter.inc(); @@ -327,7 +328,7 @@ public void txn_nested_11() { Txn.exec(counter, TxnType.WRITE, ()->{}); }); } - + @Test(expected=JenaTransactionException.class) public void txn_nested_12() { Txn.exec(counter, TxnType.READ_PROMOTE, ()->{ @@ -339,7 +340,7 @@ public void txn_nested_12() { Txn.exec(counter, TxnType.WRITE, ()->{}); }); } - + @Test public void txn_nested_13() { Txn.exec(counter, TxnType.READ_COMMITTED_PROMOTE, ()->{ @@ -378,34 +379,34 @@ public void txn_threaded_01() { @Test public void txn_threaded_02() { //Transactional tx = DatasetGraphFactory.createTxnMem(); - Transactional tx = counter; - + Transactional tx = counter; + // Start and enter the W transaction. ThreadAction a = ThreadTxn.threadTxnWrite(tx, ()->{}); // ThreadAction started ... in W transaction. Txn.exec(tx, TxnType.READ_PROMOTE, ()->{ // ... have the thread action complete. - a.run(); + a.run(); // Blocks promotion. boolean b = tx.promote(); assertFalse(b); assertEquals(ReadWrite.READ, tx.transactionMode()); }); } - + @Test public void txn_threaded_03() { Transactional tx = DatasetGraphFactory.createTxnMem(); - //Transactional tx = counter; - + //Transactional tx = counter; + // Start and enter the W transaction. ThreadAction a = ThreadTxn.threadTxnWriteAbort(tx, ()->{}); // ThreadAction started ... in W transaction. Txn.exec(tx, TxnType.READ_PROMOTE, ()->{ // ... have the thread action abort.. - a.run(); + a.run(); // Does not block promotion. boolean b = tx.promote(); assertTrue(b); @@ -413,6 +414,32 @@ public void txn_threaded_03() { }); } + @Test public void autoTxn_write_01() { + long actualValue; + try (AutoTxn txn = Txn.autoTxn(counter, TxnType.WRITE)) { + counter.inc() ; + assertEquals("In W, value()", 0, counter.value()) ; + assertEquals("In W, get()",1, counter.get()) ; + actualValue = counter.get() ; + counter.commit() ; + } + long expectedValue = counter.get(); + assertEquals("Outside W", expectedValue, actualValue) ; + } + + @Test public void autoTxn_write_02() { + long expectedValue = counter.get(); + try (AutoTxn txn = Txn.autoTxn(counter, TxnType.WRITE)) { + counter.inc() ; + assertEquals("In W, value()", 0, counter.value()) ; + assertEquals("In W, get()",1, counter.get()) ; + // Intermediate value will be reverted. + long intermediateValue = counter.get() ; + assertNotEquals(expectedValue, intermediateValue); + // no commit - auto-close is expected to abort. + } + long actualValue = counter.get(); + assertEquals("Outside W", expectedValue, actualValue) ; + } } - \ No newline at end of file diff --git a/jena-benchmarks/jena-benchmarks-jmh/pom.xml b/jena-benchmarks/jena-benchmarks-jmh/pom.xml index 958e0bc3380..8204b6963b6 100644 --- a/jena-benchmarks/jena-benchmarks-jmh/pom.xml +++ b/jena-benchmarks/jena-benchmarks-jmh/pom.xml @@ -71,6 +71,13 @@ test + + org.apache.jena + jena-geosparql + 5.5.0-SNAPSHOT + test + + org.apache.jena jena-arq diff --git a/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/BenchmarkSpatialIndex.java b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/BenchmarkSpatialIndex.java new file mode 100644 index 00000000000..e49902adb76 --- /dev/null +++ b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/BenchmarkSpatialIndex.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; +import org.apache.jena.geosparql.spatial.index.v2.GeometryGenerator; +import org.apache.jena.geosparql.spatial.index.v2.GeometryGenerator.GeometryType; +import org.apache.jena.graph.Graph; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.riot.RDFFormat; +import org.apache.jena.sparql.graph.GraphFactory; +import org.locationtech.jts.geom.Envelope; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +/** + * Benchmarking of the spatial index. + * Evaluates the time it takes to load an index from disk. + */ +@State(Scope.Benchmark) +public class BenchmarkSpatialIndex { + @Param({ + "current", + "5.1.0" + }) + public String param0_jenaVersion; + + @Param({ + "1000", + "10000", + "100000", + }) + public long param1_geometryMixes; + + @Param({ + // "", + SRS_URI.DEFAULT_WKT_CRS84 + }) + public String param2_srs; + + private SpatialIndexLifeCycle spatialIndexLifeCycle; + + @Benchmark + public void load() throws Exception { + spatialIndexLifeCycle.load(); + } + + @Setup(Level.Trial) + public void setupTrial() throws Exception { + Envelope envelope = new Envelope(-175, 175, -85, 85); + Map config = GeometryGenerator.createConfig(param1_geometryMixes); + Graph graph = GraphFactory.createDefaultGraph(); + GeometryGenerator.generateGraph(graph, envelope, config); + + String data; + RDFFormat fmt = RDFFormat.TURTLE_PRETTY; + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + RDFDataMgr.write(out, graph, fmt); + out.flush(); + data = new String(out.toByteArray(), StandardCharsets.UTF_8); + } + + String srs = param2_srs.isEmpty() ? null : param2_srs; + + switch (param0_jenaVersion) { + case "current": + spatialIndexLifeCycle = SpatialIndexCurrent.setup(data, envelope, srs, false); + break; + case "5.1.0": + spatialIndexLifeCycle = SpatialIndex510.setup(data, envelope, srs, false); + break; + default: + throw new RuntimeException("No task registered for this jena version:" + param0_jenaVersion); + } + + spatialIndexLifeCycle.init(); + spatialIndexLifeCycle.findSrs(); + spatialIndexLifeCycle.build(); + } + + @TearDown(Level.Trial) + public void tearDownTrial() throws Exception { + spatialIndexLifeCycle.close(); + } + + public static ChainedOptionsBuilder getDefaults(Class c) { + return new OptionsBuilder() + // Specify which benchmarks to run. + // You can be more specific if you'd like to run only one benchmark per test. + .include(c.getName()) + // Set the following options as needed + .mode(Mode.AverageTime) + .timeUnit(TimeUnit.SECONDS) + .warmupTime(TimeValue.NONE) + .warmupIterations(5) + .measurementIterations(5) + .measurementTime(TimeValue.NONE) + .threads(1) + .forks(1) + .shouldFailOnError(true) + .shouldDoGC(true) + //.jvmArgs("-XX:+UnlockDiagnosticVMOptions", "-XX:+PrintInlining") + .jvmArgs("-Xmx8G") + //.addProfiler(WinPerfAsmProfiler.class) + .resultFormat(ResultFormatType.JSON) + .result(c.getSimpleName() + "_" + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) + ".json"); + } + + public static void main(String[] args) throws RunnerException { + Options opt = getDefaults(BenchmarkSpatialIndex.class).build(); + new Runner(opt).run(); + } +} diff --git a/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndex510.java b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndex510.java new file mode 100644 index 00000000000..69243681b05 --- /dev/null +++ b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndex510.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.shadedJena510.geosparql.configuration.GeoSPARQLOperations; +import org.apache.shadedJena510.geosparql.spatial.SpatialIndex; +import org.apache.shadedJena510.geosparql.spatial.SpatialIndexException; +import org.apache.shadedJena510.query.Dataset; +import org.apache.shadedJena510.riot.Lang; +import org.apache.shadedJena510.riot.RDFParserBuilder; +import org.junit.Assert; +import org.locationtech.jts.geom.Envelope; + +public class SpatialIndex510 + implements SpatialIndexLifeCycle +{ + protected Dataset ds; + protected Envelope envelope; + protected String srs; + protected boolean validate; + + public SpatialIndex510(Dataset ds, Envelope envelope, String srs, boolean validate) { + super(); + this.ds = ds; + this.envelope = envelope; + this.validate = validate; + this.srs = srs; + } + + public static SpatialIndex510 setup(String data, Envelope envelope, String srs, boolean validate) throws SpatialIndexException { + Dataset ds = RDFParserBuilder.create().fromString(data).lang(Lang.TURTLE).toDataset(); + return new SpatialIndex510(ds, envelope, srs, validate); + } + + protected Path indexFile; + protected String finalSrs = null; + protected SpatialIndex indexA; + protected SpatialIndex indexB; + + @Override + public void init() { + ds.getContext().remove(SpatialIndex.SPATIAL_INDEX_SYMBOL); + } + + @Override + public void findSrs() { + finalSrs = srs == null + ? GeoSPARQLOperations.findModeSRS(ds) + : srs; + } + + @Override + public void build() throws Exception { + indexFile = Files.createTempFile("jena-", ".spatial-index"); + Files.deleteIfExists(indexFile); // buildSpatialIndex in v1 will attempt to load the file first + + indexA = SpatialIndex.buildSpatialIndex(ds, finalSrs, indexFile.toFile()); + } + + @Override + public void load() throws Exception { + indexB = SpatialIndex.load(indexFile.toFile()); + } + + @Override + public void close() throws Exception { + Files.deleteIfExists(indexFile); + + if (validate) { + int itemCountA = indexA.query(envelope).size(); + int itemCountB = indexB.query(envelope).size(); + // Assert.assertTrue(itemCountA > 0); + Assert.assertEquals(itemCountA, itemCountB); + } + } +} diff --git a/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndexCurrent.java b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndexCurrent.java new file mode 100644 index 00000000000..0a9f25afd0a --- /dev/null +++ b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndexCurrent.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.jena.geosparql.configuration.GeoSPARQLOperations; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexIoKryo; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexPerGraph; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; +import org.apache.jena.query.DatasetFactory; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParserBuilder; +import org.apache.jena.sparql.core.DatasetGraph; +import org.junit.Assert; +import org.locationtech.jts.geom.Envelope; + +public class SpatialIndexCurrent + implements SpatialIndexLifeCycle +{ + protected DatasetGraph dsg; + protected Envelope envelope; + protected String srs; + protected boolean validate; + + public SpatialIndexCurrent(DatasetGraph dsg, Envelope envelope, String srs, boolean validate) { + super(); + this.dsg = dsg; + this.envelope = envelope; + this.validate = validate; + this.srs = srs; + } + + public static SpatialIndexCurrent setup(String data, Envelope envelope, String srs, boolean validate) throws SpatialIndexException { + DatasetGraph dsg = RDFParserBuilder.create().fromString(data).lang(Lang.TURTLE).toDatasetGraph(); + return new SpatialIndexCurrent(dsg, envelope, srs, validate); + } + + protected Path indexFile; + protected String finalSrs = null; + protected SpatialIndexPerGraph indexA; + protected SpatialIndexPerGraph indexB; + + @Override + public void init() { + SpatialIndexLib.setSpatialIndex(dsg.getContext(), null); + } + + @Override + public void findSrs() { + finalSrs = srs == null + ? GeoSPARQLOperations.findModeSRS(DatasetFactory.wrap(dsg)) + : srs; + } + + @Override + public void build() throws Exception { + indexFile = Files.createTempFile("jena-", ".spatial-index"); + indexA = (SpatialIndexPerGraph)SpatialIndexLib.buildSpatialIndex(dsg, finalSrs); + SpatialIndexIoKryo.save(indexFile, indexA); + } + + @Override + public void load() throws Exception { + indexB = SpatialIndexIoKryo.load(indexFile); + } + + @Override + public void close() throws Exception { + Files.deleteIfExists(indexFile); + + if (validate) { + int itemCountA = indexA.query(envelope, null).size(); + int itemCountB = indexB.query(envelope, null).size(); + // Assert.assertTrue(itemCountA > 0); + Assert.assertEquals(itemCountA, itemCountB); + } + } +} diff --git a/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndexLifeCycle.java b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndexLifeCycle.java new file mode 100644 index 00000000000..443e8885c1e --- /dev/null +++ b/jena-benchmarks/jena-benchmarks-jmh/src/test/java/org/apache/jena/geosparql/spatial/index/SpatialIndexLifeCycle.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index; + +public interface SpatialIndexLifeCycle { + void init() throws Exception; + void findSrs() throws Exception; + void build() throws Exception; + void load() throws Exception; + void close() throws Exception; +} diff --git a/jena-benchmarks/jena-benchmarks-shadedJena510/pom.xml b/jena-benchmarks/jena-benchmarks-shadedJena510/pom.xml index ee141a3d9b2..28fa89c783b 100644 --- a/jena-benchmarks/jena-benchmarks-shadedJena510/pom.xml +++ b/jena-benchmarks/jena-benchmarks-shadedJena510/pom.xml @@ -71,6 +71,11 @@ jena-arq 5.1.0 + + org.apache.jena + jena-geosparql + 5.1.0 + diff --git a/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/build/FusekiConfig.java b/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/build/FusekiConfig.java index aff81132136..63eb901bf58 100644 --- a/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/build/FusekiConfig.java +++ b/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/build/FusekiConfig.java @@ -433,7 +433,7 @@ public static DataAccessPoint buildDataAccessPoint(Graph configuration, Node fus return dataAccess; } catch (FusekiException ex) { Fuseki.configLog.error("Skipping: Failed to build service for "+BuildLib.displayStr(configuration, n)); - Fuseki.configLog.error(" "+ex.getMessage()); + Fuseki.configLog.error(" "+ex.getMessage(), ex); return null; } } diff --git a/jena-fuseki2/jena-fuseki-geosparql/pom.xml b/jena-fuseki2/jena-fuseki-geosparql/pom.xml index 44102f7d355..2d3dcf545ad 100644 --- a/jena-fuseki2/jena-fuseki-geosparql/pom.xml +++ b/jena-fuseki2/jena-fuseki-geosparql/pom.xml @@ -25,7 +25,7 @@ jena-fuseki 5.5.0-SNAPSHOT - + GeoSPARQL with Fuseki @@ -71,20 +71,22 @@ jul-to-slf4j + org.apache.logging.log4j - log4j-slf4j2-impl + log4j-api - org.apache.logging.log4j - log4j-api + log4j-core + test org.apache.logging.log4j - log4j-core + log4j-slf4j2-impl + test diff --git a/jena-fuseki2/jena-fuseki-geosparql/src/main/java/org/apache/jena/fuseki/geosparql/DatasetOperations.java b/jena-fuseki2/jena-fuseki-geosparql/src/main/java/org/apache/jena/fuseki/geosparql/DatasetOperations.java index 765f621c5f9..decf66e1e90 100644 --- a/jena-fuseki2/jena-fuseki-geosparql/src/main/java/org/apache/jena/fuseki/geosparql/DatasetOperations.java +++ b/jena-fuseki2/jena-fuseki-geosparql/src/main/java/org/apache/jena/fuseki/geosparql/DatasetOperations.java @@ -228,10 +228,10 @@ private static void prepareSpatialExtension(Dataset dataset, ArgsConfig argsConf if (!isEmpty) { if (argsConfig.getSpatialIndexFile() != null) { File spatialIndexFile = argsConfig.getSpatialIndexFile(); - GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile); + GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath()); } else if (argsConfig.isTDBFileSetup()) { File spatialIndexFile = new File(argsConfig.getTdbFile(), SPATIAL_INDEX_FILE); - GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile); + GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath()); } else { GeoSPARQLConfig.setupSpatialIndex(dataset); } diff --git a/jena-fuseki2/jena-fuseki-main/pom.xml b/jena-fuseki2/jena-fuseki-main/pom.xml index e3c17baa7ff..a8fd7e53b5b 100644 --- a/jena-fuseki2/jena-fuseki-main/pom.xml +++ b/jena-fuseki2/jena-fuseki-main/pom.xml @@ -26,7 +26,7 @@ org.apache.jena jena-fuseki 5.5.0-SNAPSHOT - + jar @@ -34,7 +34,7 @@ org.apache.jena.fuseki.main - + org.apache.jena @@ -92,7 +92,7 @@ org.eclipse.jetty jetty-security - + org.eclipse.jetty @@ -104,13 +104,13 @@ org.apache.shiro shiro-core - + org.apache.shiro shiro-web jakarta - + org.junit.vintage @@ -129,14 +129,14 @@ junit-jupiter-params test - + org.junit.platform junit-platform-suite test - +--> @@ -172,7 +172,7 @@ test pom - + @@ -198,11 +198,11 @@ org.apache.maven.plugins maven-source-plugin - + - attach-sources-test + attach-sources-test - test-jar-no-fork + test-jar-no-fork diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/.gitignore b/jena-fuseki2/jena-fuseki-mod-geosparql/.gitignore new file mode 100644 index 00000000000..b83d22266ac --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/pom.xml b/jena-fuseki2/jena-fuseki-mod-geosparql/pom.xml new file mode 100644 index 00000000000..4dce750138e --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/pom.xml @@ -0,0 +1,169 @@ + + + + + 4.0.0 + + org.apache.jena + jena-fuseki + 5.5.0-SNAPSHOT + + + jena-fuseki-mod-geosparql + + Apache Jena - Fuseki MOD - GeoSPARQL + GeoSPARQL Module for Fuseki + + + org.apache.jena.fuseki.mod.geosparql + + + + + + org.apache.jena + jena-geosparql + 5.5.0-SNAPSHOT + + + + org.apache.jena + jena-fuseki-main + 5.5.0-SNAPSHOT + + + + + + org.junit.vintage + junit-vintage-engine + test + + + + org.junit.jupiter + junit-jupiter + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + org.junit.platform + junit-platform-suite + test + + + + org.apache.logging.log4j + log4j-core + test + + + + org.apache.logging.log4j + log4j-slf4j2-impl + test + + + + org.seleniumhq.selenium + selenium-java + test + + + + io.github.bonigarcia + webdrivermanager + test + + + + + + bundle + + + + org.apache.jena + jena-fuseki-main + 5.5.0-SNAPSHOT + provided + + + + + org.apache.jena + jena-arq + 5.5.0-SNAPSHOT + provided + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + + false + + + + + *:* + + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/DEPENDENCIES + META-INF/MANIFEST.MF + **/module-info.class + META-INF/versions/9/OSGI-INF/MANIFEST.MF + + + + + + + package + + + shade + + + + + + + + + + diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/java/org/apache/jena/fuseki/mod/geosparql/FMod_SpatialIndexer.java b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/java/org/apache/jena/fuseki/mod/geosparql/FMod_SpatialIndexer.java new file mode 100644 index 00000000000..b17f50925af --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/java/org/apache/jena/fuseki/mod/geosparql/FMod_SpatialIndexer.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.fuseki.mod.geosparql; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.auth.AuthPolicy; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.sys.FusekiAutoModule; +import org.apache.jena.fuseki.server.DataAccessPoint; +import org.apache.jena.fuseki.server.DataAccessPointRegistry; +import org.apache.jena.fuseki.server.DataService; +import org.apache.jena.fuseki.server.Endpoint; +import org.apache.jena.fuseki.server.Operation; +import org.apache.jena.rdf.model.Model; + +public class FMod_SpatialIndexer implements FusekiAutoModule { + + public static Operation spatialIndexerOperation = + Operation.alloc("http://jena.apache.org/fuseki#spatial-indexer", + "spatial-indexer", + "Spatial indexer service"); + + public FMod_SpatialIndexer() { + super(); + } + + @Override + public String name() { + return "Spatial Indexer"; + } + + @Override + public void start() { + } + + @Override + public void prepare(FusekiServer.Builder builder, Set datasetNames, Model configModel) { + Fuseki.configLog.info(name() + ": Registering operation " + spatialIndexerOperation.getId()); + builder.registerOperation(spatialIndexerOperation, new SpatialIndexerService()); + } + + /** + * The spatial indexer endpoint is created for every SPARQL update endpoint. + * The update endpoint's auth policy is inherited. + */ + @Override + public void configured(FusekiServer.Builder serverBuilder, DataAccessPointRegistry dapRegistry, Model configModel) { + FusekiAutoModule.super.configured(serverBuilder, dapRegistry, configModel); + + boolean autoConfigure = false; + if (autoConfigure) { + autoConfigure(serverBuilder, dapRegistry, configModel); + } + } + + /** + * Disabled for now. + * + * Automatically creates a corresponding spatial indexer endpoint for each SPARQL update endpoint. + * The spatial indexer endpoint with name follows the pattern '{updateEndpointName}-spatial and inherits the update endpoint's auth policy. + */ + private void autoConfigure(FusekiServer.Builder serverBuilder, DataAccessPointRegistry dapRegistry, Model configModel) { + FusekiAutoModule.super.configured(serverBuilder, dapRegistry, configModel); + + List newDataAccessPoints = new ArrayList<>(); + // Register the spatial indexer for each update endpoint and inherit its auth policy. + for (DataAccessPoint dap : dapRegistry.accessPoints()) { + String dapName = dap.getName(); + String logPrefix = name() + " - " + dapName + ": "; + + DataService dataService = dap.getDataService(); + List updateEndpoints = Optional.ofNullable(dataService.getEndpoints(Operation.Update)) + .orElse(List.of()); + + List spatialEndpoints = new ArrayList<>(); + for (Endpoint updateEndpoint : updateEndpoints) { + String updateEndpointName = updateEndpoint.getName(); + String geoIndexerEndpointName = updateEndpointName + "-spatial"; + AuthPolicy authPolicy = updateEndpoint.getAuthPolicy(); + + Fuseki.configLog.info(logPrefix + "Registering spatial indexer endpoint: " + geoIndexerEndpointName); + + Endpoint geoIndexerEndpoint = Endpoint.create() + .operation(spatialIndexerOperation) + .endpointName(geoIndexerEndpointName) + .authPolicy(authPolicy) + .build(); + + spatialEndpoints.add(geoIndexerEndpoint); + } + + // Create new DataService based on existing one with the spatial indexer endpoints attached. + DataService.Builder dSrvBuilder = DataService.newBuilder(dataService); + spatialEndpoints.forEach(dSrvBuilder::addEndpoint); + DataService newDataService = dSrvBuilder.build(); + newDataAccessPoints.add(new DataAccessPoint(dapName, newDataService)); + } + + // "replace" each DataAccessPoint + newDataAccessPoints.forEach(dap -> { + dapRegistry.remove(dap.getName()); + dapRegistry.register(dap); + }); + } +} diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/java/org/apache/jena/fuseki/mod/geosparql/SpatialIndexerService.java b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/java/org/apache/jena/fuseki/mod/geosparql/SpatialIndexerService.java new file mode 100644 index 00000000000..87eaa33aa10 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/java/org/apache/jena/fuseki/mod/geosparql/SpatialIndexerService.java @@ -0,0 +1,625 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.geosparql; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.jena.atlas.io.IO; +import org.apache.jena.atlas.lib.DateTimeUtils; +import org.apache.jena.fuseki.server.Endpoint; +import org.apache.jena.fuseki.servlets.BaseActionREST; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexConstants; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexPerGraph; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexerComputation; +import org.apache.jena.geosparql.spatial.task.BasicTask; +import org.apache.jena.geosparql.spatial.task.BasicTask.TaskListener; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.query.Query; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.riot.WebContent; +import org.apache.jena.riot.web.HttpNames; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.sparql.core.Var; +import org.apache.jena.sparql.engine.binding.Binding; +import org.apache.jena.sparql.exec.QueryExec; +import org.apache.jena.sparql.exec.RowSet; +import org.apache.jena.sparql.syntax.syntaxtransform.QueryTransformOps; +import org.apache.jena.sparql.util.Context; +import org.apache.jena.system.Txn; +import org.apache.jena.web.HttpSC; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonWriter; + +import jakarta.servlet.AsyncContext; +import jakarta.servlet.AsyncEvent; +import jakarta.servlet.AsyncListener; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +/** + * Spatial index (re)computation service. + * + * Supports two types of tasks that are executed concurrently. For a given data set, only a single + * task can be active at a given time. The status of the most recent task can be queried via the + * rest API. Task execution also broadcasts start/abort/termination events via server side events + * (SSE). + * + *
    + *
  • Updating/replacing the graphs of a spatial index.
  • + *
  • Removal of graphs from a spatial index that are absent in the corresponding data set.
  • + *
+ */ +public class SpatialIndexerService extends BaseActionREST { + private static final Logger logger = LoggerFactory.getLogger(SpatialIndexerService.class); + + /** Help class for SSE listeners per endpoint. */ + private class EndpointClients { + Map eventListeners = Collections.synchronizedMap(new IdentityHashMap<>()); + } + + /** Gson configured to not emit newlines (pretty printing = off). */ + private static Gson gsonForSse = new Gson(); + + /** Registered clients listening to server side events for indexer status updates. */ + private Map listenersByEndpoint = Collections.synchronizedMap(new IdentityHashMap<>()); + + + /** Constants for serving graph listings via sparql. */ + private static final Var graphVar = Var.alloc("g"); + private static final Var keywordVar = Var.alloc("keyword"); + private static final Query allGraphsQuery = QueryFactory.create("SELECT ?g { GRAPH ?g { } } ORDER BY ASC(?g)"); + private static final Query graphsByKeywordQuery = QueryFactory.create("SELECT ?g { GRAPH ?g { } FILTER(contains(lcase(str(?g)), ?keyword)) } ORDER BY ASC(?g)"); + + public SpatialIndexerService() {} + + private static > Set extractGraphsFromRequest(DatasetGraph dsg, HttpAction action) { + String uris = action.getRequest().getParameter(HttpNames.paramGraph); + Collection strs; + if (uris == null || uris.isBlank()) { + strs = List.of(Quad.defaultGraphIRI.toString(), Quad.unionGraph.toString()); + } else { + TypeToken> typeToken = new TypeToken>(){}; + strs = gsonForSse.fromJson(uris, typeToken); + } + List rawGraphNodes = strs.stream().map(NodeFactory::createURI).distinct().toList(); + // If the set of specified graphs is empty then index all. + if (rawGraphNodes.isEmpty()) { + rawGraphNodes = List.of(Quad.defaultGraphIRI, Quad.unionGraph); + } + + Set uniqueGraphNodes = rawGraphNodes.stream() + .flatMap(node -> expandUnionGraphNode(new ArrayList<>(), dsg, node).stream()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + return uniqueGraphNodes; + } + + private static boolean isReplaceMode(HttpAction action) { + String str = action.getRequest().getParameter("replaceMode"); + boolean result = (str == null || str.isBlank()) ? false : Boolean.parseBoolean(str); + return result; + } + + private static int getThreadCount(HttpAction action) { + String str = action.getRequest().getParameter("maxThreadCount"); + int result = (str == null || str.isBlank()) ? 1 : Integer.parseInt(str); + + if (result == 0) { + result = Runtime.getRuntime().availableProcessors(); + } + + return result; + } + + private static > C expandUnionGraphNode(C accGraphs, DatasetGraph dsg, Node node) { + if (Quad.isUnionGraph(node)) { + SpatialIndexLib.accGraphNodes(accGraphs, dsg); + } else { + accGraphs.add(node); + } + return accGraphs; + } + + /** + * The GET command can serve: the website, the notification stream from task execution + * and the latest task execution status. + */ + @Override + protected void doGet(HttpAction action) { + String rawCommand = action.getRequestParameter("command"); + String command = Optional.ofNullable(rawCommand).orElse("webpage"); + switch (command) { + case "webpage": serveWebPage(action); break; + case "events": serveEvents(action); break; + case "status": serveStatus(action); break; + default: + throw new UnsupportedOperationException("Unsupported command (via HTTP GET): " + command); + } + } + + protected void serveWebPage(HttpAction action) { + // Serves the minimal graphql ui + String resourceName = "spatial-indexer/index.html"; + String str = null; + try (InputStream in = SpatialIndexerService.class.getClassLoader().getResourceAsStream(resourceName)) { + str = IOUtils.toString(in, StandardCharsets.UTF_8); + } catch (IOException e) { + ServletOps.errorOccurred(e.getMessage(), e); + } + + if (str == null) { + ServletOps.error(HttpSC.INTERNAL_SERVER_ERROR_500, "Failed to load classpath resource " + resourceName); + } else { + action.setResponseStatus(HttpSC.OK_200); + action.setResponseContentType(WebContent.contentTypeHTML); + try (OutputStream out = action.getResponseOutputStream()) { + IOUtils.write(str, out, StandardCharsets.UTF_8); + } catch (IOException e) { + ServletOps.errorOccurred(e); + } + } + } + + protected BasicTask getActiveTask(HttpAction action) { + DatasetGraph dsg = action.getDataset(); + Context cxt = dsg.getContext(); + BasicTask activeTask = cxt.get(SpatialIndexConstants.symSpatialIndexTask); + return activeTask; + } + + /** + * Post request: Handle API call. + * Request is rejected if there is an already running task. + */ + @Override + protected void doPost(HttpAction action) { + try { + String rawCommand = action.getRequestParameter("command"); + String command = Optional.ofNullable(rawCommand).orElse("none"); + switch (command) { + case "index": doIndex(action); break; + case "clean": doClean(action); break; + case "status": serveStatus(action); break; + case "cancel": doCancel(action); break; + case "graphs": serveGraphs(action); break; + default: + throw new UnsupportedOperationException("Unsupported command (via HTTP POST): " + command); + } + } catch (Throwable t) { + action.log.error("An unexpected error occurred.", t); + ServletOps.errorOccurred(t); + } + } + + /** + * Serves a JSON array of the graph IRIs visible to the authenticated user. + * Non-IRI graph names are omitted in the output. + * + * Note: It is intentional to NOT expose a full SPARQL endpoint for security reasons. + */ + protected void serveGraphs(HttpAction action) throws IOException { + long offset = Optional.ofNullable(action.getRequestParameter("offset")).map(Long::parseLong).orElse(Query.NOLIMIT); + long limit = Optional.ofNullable(action.getRequestParameter("limit")).map(Long::parseLong).orElse(Query.NOLIMIT); + String keyword = action.getRequestParameter("keyword"); + + Query query; + if (keyword != null) { + query = graphsByKeywordQuery.cloneQuery(); + query = QueryTransformOps.syntaxSubstitute(query, Map.of(keywordVar, NodeFactory.createLiteralString(keyword))); + } else { + query = allGraphsQuery.cloneQuery(); + } + + if (offset != Query.NOLIMIT || limit != Query.NOLIMIT) { + query.setLimit(limit); + query.setOffset(offset); + } + + action.beginRead(); + try { + DatasetGraph dsg = action.getActiveDSG(); + // XXX Make tracked should the exec tracker PR become available. + try (QueryExec qe = QueryExec.dataset(dsg).query(query).build()) { + try (JsonWriter jsonWriter = gsonForSse.newJsonWriter( + new OutputStreamWriter(action.getResponseOutputStream(), StandardCharsets.UTF_8))) { + RowSet rs = qe.select(); + jsonWriter.beginArray(); + jsonWriter.value(Quad.defaultGraphIRI.getURI()); + jsonWriter.value(Quad.unionGraph.getURI()); + while (rs.hasNext()) { + Binding b = rs.next(); + Node n = b.get(graphVar); + if (n.isURI()) { + String uri = n.getURI(); + jsonWriter.value(uri); + } + } + jsonWriter.endArray(); + jsonWriter.flush(); + } + } + } finally { + action.endRead(); + } + } + + protected void serveEvents(HttpAction action) { + HttpServletRequest request = action.getRequest(); + HttpServletResponse response = action.getResponse(); + + response.setContentType("text/event-stream"); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Cache-Control", "no-cache"); + + final AsyncContext asyncContext = request.startAsync(); + asyncContext.setTimeout(0); + + Endpoint endpoint = action.getEndpoint(); + + Runnable[] disposeSseListener = {null}; + + // Detect when client disconnects + asyncContext.addListener(new AsyncListener() { + @Override + public void onComplete(AsyncEvent event) { + disposeSseListener[0].run(); + } + + @Override + public void onTimeout(AsyncEvent event) { + disposeSseListener[0].run(); + } + + @Override + public void onError(AsyncEvent event) { + disposeSseListener[0].run(); + } + + @Override + public void onStartAsync(AsyncEvent event) { + // No-op + } + }); + + disposeSseListener[0] = () -> { + listenersByEndpoint.compute(endpoint, (et, clts) -> { + EndpointClients r = clts; + if (clts != null) { + // Remove the listener for the async context. + // If no more listeners remain then dispose the exec tracker listener. + clts.eventListeners.remove(asyncContext); + if (clts.eventListeners.isEmpty()) { + r = null; + } + return r; + } + return r; + }); + }; + + // Atomically set up the new listener. + listenersByEndpoint.compute(endpoint, (et, clients) -> { + if (clients == null) { + clients = new EndpointClients(); + } + clients.eventListeners.put(asyncContext, disposeSseListener[0]); + return clients; + }); + } + + /** + * Remove all graphs from the index for which there is no corresponding graph in the dataset. + */ + protected void doClean(HttpAction action) throws Exception { + DatasetGraph dsg = action.getDataset(); + Endpoint endpoint = action.getEndpoint(); + + TaskListener taskListener = task -> { + switch (task.getTaskState()) { + case STARTING: { + JsonObject json = toJsonTaskStart(task.getStartTime(), null); + broadcastJson(endpoint, json); + break; + } + case TERMINATED: { + JsonObject json = toJsonTaskEnd(task.getEndTime(), task.getThrowable(), task.getStatusMessage()); + broadcastJson(endpoint, json); + break; + } + default: + break; + } + }; + + try { + SpatialIndexLib.scheduleOnceCleanTask(dsg, taskListener); + } catch (Exception e) { + ServletOps.errorOccurred(e.getMessage(), e); + } + } + + /** Send a stop request to a running task. Does not wait for the task to terminate. */ + protected void doCancel(HttpAction action) { + BasicTask task = getActiveTask(action); + String state; + if (task != null) { + state = "true"; + task.abort(); + } else { + state = "false"; + } + + String jsonStr = String.format("{ \"stopped\": %s }", state); + successJson(action, jsonStr); + } + + /** + * Serves a JSON object that captures the current indexing status. + * + *
+     * {
+     *   isIndexing: Boolean, // Whether an index task is running.
+     *   isAborting: Boolean, // Iff isIndexing==true: Whether abort has been requested.
+     *   error: String,       // Iff isIndexing==false: An error message of the most recently terminated indexing task.
+     *   message: String      // Status message of the most recent indexing task (running or terminated).
+     *   time: long           // EpochMillis of the last status update.
+     * }
+     * 
+ */ + protected void serveStatus(HttpAction action) { + BasicTask task = getActiveTask(action); + + JsonObject status = new JsonObject(); + long time; + if (task == null) { + status.addProperty("isIndexing", false); + time = 0; + } else { + status.addProperty("isIndexing", !task.isTerminated()); + if (!task.isTerminated()) { + status.addProperty("isAborting", task.isAborting()); + time = !task.isAborting() ? task.getStartTime() : task.getAbortTime(); + } else { + time = task.getEndTime(); + } + Throwable throwable = task.getThrowable(); + if (throwable != null) { + String msg = ExceptionUtils.getStackTrace(throwable); + status.addProperty("error", msg); + } + + String msg = task.getStatusMessage(); + if (msg != null) { + status.addProperty("message", msg); + } + } + status.addProperty("time", time); + + + String jsonStr = gsonForSse.toJson(status); + successJson(action, jsonStr); + } + + protected BasicTask scheduleIndexTask(HttpAction action, SpatialIndexerComputation indexComputation, Path targetFile, boolean isReplaceTask) { + Endpoint endpoint = action.getEndpoint(); + DatasetGraph dsg = action.getDataset(); + + long graphCount = indexComputation.getGraphNodes().size(); + + TaskListener taskListener = new TaskListener() { + @Override + public void onStateChange(BasicTask task) { + switch (task.getTaskState()) { + case STARTING: { + JsonObject json = toJsonTaskStart(task.getStartTime(), null); + broadcastJson(endpoint, json); + break; + } + case ABORTING: { + JsonObject json = toJsonTaskAbort(task.getAbortTime(), null); + broadcastJson(endpoint, json); + break; + } + case TERMINATED: { + Throwable throwable = task.getThrowable(); + long endTime = task.getEndTime(); + JsonObject json = toJsonTaskEnd(endTime, throwable, task.getStatusMessage()); + broadcastJson(endpoint, json); + if (logger.isInfoEnabled()) { + logger.info("Indexing task of {} graphs terminated.", graphCount); + } + } + default: + break; + } + } + }; + + return SpatialIndexLib.scheduleOnceIndexTask(dsg, indexComputation, targetFile, isReplaceTask, taskListener); + } + + protected void doIndex(HttpAction action) throws Exception { + DatasetGraph dsg = action.getDataset(); + SpatialIndex index = SpatialIndexLib.getSpatialIndex(dsg.getContext()); + + if (index == null) { // error: no spatial index has been configured + // XXX Could still allow for creating an ad-hoc in-memory-only index. + String msg = String.format("[%d] No spatial index has been configured for the dataset", action.id); + action.log.error(msg); + ServletOps.error(HttpSC.SERVICE_UNAVAILABLE_503, msg); + } else { + boolean isReplaceMode = isReplaceMode(action); + int threadCount = getThreadCount(action); + + // Only SpatialIndexPerGraph can be updated. + // Check if the index can be updated. + // If not then raise an exception + // that informs that only replace mode can be used in this situation. + if (!(index instanceof SpatialIndexPerGraph)) { + boolean isUpdateMode = !isReplaceMode; + if (isUpdateMode) { + throw new RuntimeException("Cannot update existing spatial index because its type is unsupported. Consider replacing the index."); + } + } + + Path oldLocation = index.getLocation(); + if (oldLocation == null) { + action.log.warn("Spatial index will not be persisted because no file location was configured."); + } + + String srsURI = index.getSrsInfo().getSrsURI(); + + List graphNodes = new ArrayList<>(Txn.calculateRead(dsg, () -> extractGraphsFromRequest(dsg, action))); + SpatialIndexerComputation task = new SpatialIndexerComputation(dsg, srsURI, graphNodes, threadCount); + + action.log.info(String.format("[%d] spatial index: computation request accepted.", action.id)); + + try { + scheduleIndexTask(action, task, oldLocation, isReplaceMode); + successText(action, "Spatial index computation task accepted at " + DateTimeUtils.nowAsXSDDateTimeString()); + } catch (Exception e) { + ServletOps.errorOccurred(e.getMessage(), e); + } + } + } + + protected static JsonObject toJsonTaskStart(long timeInMillis, String msg) { + JsonObject json = new JsonObject(); + json.addProperty("isIndexing", true); + json.addProperty("time", timeInMillis); + if (msg != null) { + json.addProperty("message", msg); + } + return json; + } + protected static JsonObject toJsonTaskAbort(long timeInMillis, String msg) { + JsonObject json = new JsonObject(); + json.addProperty("isIndexing", true); + json.addProperty("isAborting", true); + json.addProperty("time", timeInMillis); + if (msg != null) { + json.addProperty("message", msg); + } + return json; + } + + protected static JsonObject toJsonTaskEnd(long timeInMillis, Throwable throwable, String msg) { + JsonObject json = new JsonObject(); + json.addProperty("isIndexing", false); + json.addProperty("time", timeInMillis); + if (msg != null) { + json.addProperty("message", msg); + } + if (throwable != null) { + json.addProperty("error", ExceptionUtils.getStackTrace(throwable)); + } + return json; + } + + protected void broadcastJson(Endpoint endpoint, JsonElement jsonData) { + EndpointClients clients = listenersByEndpoint.get(endpoint); + if (clients != null) { + Iterator> it = clients.eventListeners.entrySet().iterator(); + broadcastJson(it, jsonData); + } + } + + protected void broadcastJson(Iterator> it, JsonElement jsonData) { + String str = gsonForSse.toJson(jsonData); + broadcastLine(it, str); + } + + /** + * Broadcast a payload to all listeners in the given iterator. + * On failure, the listener is removed from the iterator. + */ + protected void broadcastLine(Iterator> it, String payload) { + while (it.hasNext()) { + Entry e = it.next(); + AsyncContext context = e.getKey(); + Runnable unregister = e.getValue(); + try { + PrintWriter writer = context.getResponse().getWriter(); + // Format demanded by server side events is: "data: \n\n". + writer.println("data: " + payload); + writer.println(); + writer.flush(); + } catch (Throwable x) { + it.remove(); // Remove first so that unregister does not cause concurrent modification. + logger.warn("Broadcast failed.", x); + try { + unregister.run(); + } finally { + context.complete(); + } + } + } + } + + protected static void successText(HttpAction action, String jsonStr) { + successStringUtf8(action, WebContent.contentTypeTextPlain, jsonStr); + } + + protected static void successJson(HttpAction action, String jsonStr) { + successStringUtf8(action, WebContent.contentTypeJSON, jsonStr); + } + + protected static void successStringUtf8(HttpAction action, String contentType, String str) { + action.setResponseContentType(contentType); + action.setResponseCharacterEncoding(WebContent.charsetUTF8); + action.setResponseStatus(HttpSC.OK_200); + try { + action.getResponseOutputStream().println(str); + } catch (IOException e) { + IO.exception(e); + } + return; + } +} diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/resources/META-INF/services/org.apache.jena.fuseki.main.sys.FusekiAutoModule b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/resources/META-INF/services/org.apache.jena.fuseki.main.sys.FusekiAutoModule new file mode 100644 index 00000000000..9e6353c9738 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/resources/META-INF/services/org.apache.jena.fuseki.main.sys.FusekiAutoModule @@ -0,0 +1 @@ +org.apache.jena.fuseki.mod.geosparql.FMod_SpatialIndexer diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/resources/spatial-indexer/index.html b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/resources/spatial-indexer/index.html new file mode 100644 index 00000000000..be6caaacfb2 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/src/main/resources/spatial-indexer/index.html @@ -0,0 +1,389 @@ + + + + + + Spatial Index Controls + + + +

Rebuild Spatial Index for Selected Graphs

+ +
+
+ +
+
+

+        
+
+ +
+ + + + +
+ + +
+
+ + +
+
+ +
+
+ + + + + + + + + + + + + +
SelectGraph Name
+ + + + + diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/java/org/apache/jena/fuseki/mod/geosparql/TestFMod_SpatialIndexer.java b/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/java/org/apache/jena/fuseki/mod/geosparql/TestFMod_SpatialIndexer.java new file mode 100644 index 00000000000..30e0a449a5b --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/java/org/apache/jena/fuseki/mod/geosparql/TestFMod_SpatialIndexer.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.geosparql; + +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import org.apache.jena.atlas.iterator.Iter; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.cmds.FusekiMain; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v2.GeometryGenerator; +import org.apache.jena.geosparql.spatial.index.v2.GeometryGenerator.GeometryType; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexPerGraph; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.core.Quad; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.locationtech.jts.geom.Envelope; +import org.openqa.selenium.By; +import org.openqa.selenium.JavascriptExecutor; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.openqa.selenium.support.ui.WebDriverWait; + +import io.github.bonigarcia.wdm.WebDriverManager; + +/** + * Test cases that interact with the spatial indexer web UI via Selenium. + * + * This class is currently set to "ignore" because it requires local browser. + * Although, a headless Chrome should be started automatically, + * this step turns out to not yet work reliable across all environments. + */ +@Ignore +public class TestFMod_SpatialIndexer { + private WebDriver driver; + private JavascriptExecutor js; + + private static Envelope queryEnvelope = new Envelope(-180, 180, -90, 90); + private DatasetGraph dsg; + private SpatialIndex spatialIndex; + private Node graphName1 = NodeFactory.createURI("http://www.example.org/graph1"); + + @Before + public void setUp() throws IOException, SpatialIndexException { + dsg = DatasetGraphFactory.create(); + setupTestData(dsg); + + spatialIndex = SpatialIndexLib.buildSpatialIndex(dsg); + + String[] argv = new String[] { "--empty" }; + + FusekiServer server = FusekiMain.builder(argv) + .add("test", dsg) + .registerOperation(FMod_SpatialIndexer.spatialIndexerOperation, new SpatialIndexerService()) + .addEndpoint("test", "spatial-indexer", FMod_SpatialIndexer.spatialIndexerOperation) + .build(); + server.start(); + int port = server.getPort(); + String serverURL = "http://localhost:" + port + "/"; + String siteUrl = serverURL + "test/spatial-indexer"; + + ChromeOptions options = new ChromeOptions(); + options.addArguments("--headless=new"); // use "new" headless mode (better support) + options.addArguments("--no-sandbox"); + options.addArguments("--disable-dev-shm-usage"); + + WebDriverManager.chromedriver().setup(); // Automatically downloads and sets path + driver = new ChromeDriver(options); + driver.get(siteUrl); + js = (JavascriptExecutor) driver; + } + + private void setupTestData(DatasetGraph dsg) { + // Fill the graph with a few geometries; spatial index construction will derive the SRS from them. + Envelope envelope = new Envelope(-175, 175, -85, 85); + + Map conf = new HashMap<>(); + conf.put(GeometryType.POINT, 1); + + // Generate geometries into the default graph and a named graph + GeometryGenerator.generateGraph(dsg.getDefaultGraph(), envelope, conf); + + conf.put(GeometryType.POLYGON, 1); + GeometryGenerator.generateGraph(dsg.getGraph(graphName1), envelope, conf); + } + + /** + * Test that first clicks the "apply" button on HTML page to update the spatial index. + * Then, the test removes a graph from the dataset and clicks the "clean" button which + * should remove all entries from the index for which there is no corresponding graph in the dataset. + */ + @Test + public void testIndexAndCleanButtons() { + // Index the test data (default graph and 1 named graph). + WebElement button = driver.findElement(By.id("apply-action")); + button.click(); + awaitEvent(); + Assert.assertEquals(1, spatialIndex.query(queryEnvelope, Quad.defaultGraphIRI).size()); + Assert.assertEquals(2, spatialIndex.query(queryEnvelope, graphName1).size()); + clearLastEvent(); + + // Remove the named graph and update the index. + dsg.removeGraph(graphName1); + WebElement cleanButton = driver.findElement(By.id("clean-action")); + cleanButton.click(); + awaitEvent(); + SpatialIndexPerGraph newIndex = (SpatialIndexPerGraph)SpatialIndexLib.getSpatialIndex(dsg.getContext()); + + // Check the number of graphs in the spatial index; the treeMap includes the default graph. + long numGraphsInIndex = newIndex.getIndex().getTreeMap().keySet().size(); + long numGraphsInDataset; + try (Stream s = Iter.asStream(dsg.listGraphNodes())) { + numGraphsInDataset = s.count() + 1; // Add one for the default graph. + } + Assert.assertEquals(numGraphsInIndex, numGraphsInDataset); + clearLastEvent(); + } + + private void awaitEvent() { + WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(5)); + wait.until(d -> { + Object status = js.executeScript("return window.lastEvent;"); + return status != null; + }); + } + + private void clearLastEvent() { + js.executeScript("window.lastEvent = null"); + } + + @After + public void tearDown() { + if (driver != null) { + driver.quit(); + } + } +} diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/java/org/apache/jena/fuseki/mod/geosparql/TestSpatialIndexerTasks.java b/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/java/org/apache/jena/fuseki/mod/geosparql/TestSpatialIndexerTasks.java new file mode 100644 index 00000000000..29c91a725d1 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/java/org/apache/jena/fuseki/mod/geosparql/TestSpatialIndexerTasks.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.geosparql; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.function.Predicate; + +import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v2.GeometryGenerator; +import org.apache.jena.geosparql.spatial.index.v2.GeometryGenerator.GeometryType; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexPerGraph; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexerComputation; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.core.Quad; +import org.junit.Assert; +import org.junit.Test; +import org.locationtech.jts.geom.Envelope; + +/** Test cases that check for whether the correct graphs are indexed - also when a user is only authorized to a certain subset of graphs. */ +public class TestSpatialIndexerTasks { + + private static final Node graph(int i) { + return NodeFactory.createURI("http://www.example.org/graph" + i); + } + + private static final Node dg = Quad.defaultGraphIRI; + private static final Node g1 = graph(1); + private static final Node g2 = graph(2); + private static final Node g3 = graph(3); + private static final Node g4 = graph(4); + + private static final Set allGraphs = Set.of(Quad.defaultGraphIRI, g1, g2, g3, g4); + + /** Test data: spatial data in 1 default graph (dg) and 4 named graphs (g1, g2, g3, 4) */ + private static DatasetGraph createTestData() { + Map conf = GeometryGenerator.createConfig(1); + DatasetGraph dsg = DatasetGraphFactory.create(); + for (int i = 0; i < 5; ++i) { + Node graph = i == 0 + ? dg + : graph(i); + Graph g = dsg.getGraph(graph); + Envelope env = new Envelope(-175, 175, -85, 85); + GeometryGenerator.generateGraph(g, env, conf); + } + return dsg; + } + + /** Helper method: Replace a prior index with the given one. Returns the new spatial index. */ + private static SpatialIndexPerGraph replaceSpatialIndex(DatasetGraph dsg, Predicate isAuthorizedGraph, String srs, Collection replaceGraphs) { + List graphNodes = List.copyOf(replaceGraphs); + SpatialIndexerComputation computation = new SpatialIndexerComputation(dsg, srs, graphNodes, 1); + SpatialIndexLib.createIndexerTask(dsg, isAuthorizedGraph, computation, null, null, true).run(); + SpatialIndexPerGraph spatialIndex = SpatialIndexLib.getSpatialIndex(dsg); + SpatialIndexLib.setSpatialIndex(dsg, spatialIndex); + + // Access the spatial index via context to test the machinery. + SpatialIndexPerGraph result = SpatialIndexLib.getSpatialIndex(dsg); + return result; + } + + /** Helper method: Get or create a spatial index and update the requested graphs in it. Returns the new spatial index. */ + private static SpatialIndexPerGraph updateSpatialIndex(DatasetGraph dsg, Predicate isAuthorizedGraph, String srs, Collection updateGraphs) { + List graphNodes = List.copyOf(updateGraphs); + SpatialIndexerComputation computation = new SpatialIndexerComputation(dsg, srs, graphNodes, 1); + SpatialIndexLib.createIndexerTask(dsg, isAuthorizedGraph, computation, null, null, false).run(); + SpatialIndexPerGraph spatialIndex = SpatialIndexLib.getSpatialIndex(dsg); + return spatialIndex; + } + + /** Helper method: run a clean task and return the resulting index from the context. */ + private static SpatialIndexPerGraph clean(DatasetGraph dsg, Predicate isAuthorizedGraph) { + SpatialIndexLib.createCleanTask(dsg, isAuthorizedGraph, null).run(); + SpatialIndexPerGraph spatialIndex = SpatialIndexLib.getSpatialIndex(dsg); + return spatialIndex; + } + + @Test + public void testDsgIndexUpdate() throws SpatialIndexException { + DatasetGraph dsg = createTestData(); + + Set initialGraphs = Set.of(g2, g3, g4); + Set updateGraphs = Set.of(dg, g1); + + replaceSpatialIndex(dsg, null, SRS_URI.DEFAULT_WKT_CRS84, initialGraphs); + SpatialIndexPerGraph spatialIndex = updateSpatialIndex(dsg, null, SRS_URI.DEFAULT_WKT_CRS84, updateGraphs); + + Set indexedGraphNodes = spatialIndex.getIndex().getTreeMap().keySet(); + Assert.assertEquals(allGraphs, indexedGraphNodes); + } + + @Test + public void testDsgIndexReplace() throws SpatialIndexException { + DatasetGraph dsg = createTestData(); + + SpatialIndexLib.buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + + Set expectedGraphSet = Set.of(g1, g2); + SpatialIndexPerGraph spatialIndex = replaceSpatialIndex(dsg, null, SRS_URI.DEFAULT_WKT_CRS84, expectedGraphSet); + + Set actualGraphSet = spatialIndex.getIndex().getTreeMap().keySet(); + Assert.assertEquals(expectedGraphSet, actualGraphSet); + } + + @Test + public void testDsgIndexClean() throws InterruptedException, ExecutionException, SpatialIndexException { + DatasetGraph dsg = createTestData(); + + SpatialIndexLib.buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + + Set.of(dg, g1, g2, g3).forEach(dsg::removeGraph); + + SpatialIndexPerGraph spatialIndex = clean(dsg, null); + Set actual = spatialIndex.getIndex().getTreeMap().keySet(); + Set expected = Set.of(dg, g4);; + Assert.assertEquals(expected, actual); + } + + @Test + public void testProtectedDsgIndexUpdate() throws SpatialIndexException { + DatasetGraph dsg = createTestData(); + + // User can only see g3 and g4 and requests to update both of them. + Set authorizedGraph = Set.of(g3, g4); + Set initialGraphs = Set.of(dg, g1, g2, g3); + Set updateGraphs = Set.of(g3, g4); + + replaceSpatialIndex(dsg, null, SRS_URI.DEFAULT_WKT_CRS84, initialGraphs); + SpatialIndexPerGraph spatialIndex = updateSpatialIndex(dsg, authorizedGraph::contains, SRS_URI.DEFAULT_WKT_CRS84, updateGraphs); + + Set indexedGraphNodes = spatialIndex.getIndex().getTreeMap().keySet(); + Assert.assertEquals(allGraphs, indexedGraphNodes); + } + + @Test + public void testProtectedDsgIndexReplace() { + DatasetGraph dsg = createTestData(); + + // User can only see g3 and g4 and requests to replace index with only g4 - so g3 should get lost. + Set authorizedGraphs = Set.of(g3, g4); + + Set initialGraphs = Set.of(dg, g1, g2, g3); + Set replaceGraphs = Set.of(g4); + Set expectedGraphs = Set.of(dg, g1, g2, g4); + + replaceSpatialIndex(dsg, null, SRS_URI.DEFAULT_WKT_CRS84, initialGraphs); + SpatialIndexPerGraph spatialIndex = replaceSpatialIndex(dsg, authorizedGraphs::contains, SRS_URI.DEFAULT_WKT_CRS84, replaceGraphs); + + Set indexedGraphNodes = spatialIndex.getIndex().getTreeMap().keySet(); + Assert.assertEquals(expectedGraphs, indexedGraphNodes); + } + + @Test + public void testProtectedDsgCleanAuthorized() throws SpatialIndexException { + DatasetGraph dsg = createTestData(); + + // User is authorized for g3 and g4 - but g4 gets removed -> g4 should become removed. + Set authorizedGraphs = Set.of(g3, g4); + + SpatialIndexLib.buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + Set.of(dg, g4).forEach(dsg::removeGraph); + + SpatialIndexPerGraph spatialIndex = clean(dsg, authorizedGraphs::contains); + Set actual = spatialIndex.getIndex().getTreeMap().keySet(); + Set expected = Set.of(dg, g1, g2, g3); + Assert.assertEquals(expected, actual); + } + + @Test + public void testProtectedDsgCleanUnauthorized() throws SpatialIndexException { + DatasetGraph dsg = createTestData(); + SpatialIndexLib.buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + Set.of(dg, g1, g3).forEach(dsg::removeGraph); + + // Among the authorized graphs, only g4 has been removed and can thus be cleaned up. + Set authorizedGraphs = Set.of(g3, g4); + + SpatialIndexPerGraph spatialIndex = clean(dsg, authorizedGraphs::contains); + Set actual = spatialIndex.getIndex().getTreeMap().keySet(); + Set expected = Set.of(dg, g1, g2, g4); + Assert.assertEquals(expected, actual); + } +} diff --git a/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/resources/log4j2-test.properties b/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/resources/log4j2-test.properties new file mode 100644 index 00000000000..fb3efdb6deb --- /dev/null +++ b/jena-fuseki2/jena-fuseki-mod-geosparql/src/test/resources/log4j2-test.properties @@ -0,0 +1,56 @@ +## Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 +status = error +name = PropertiesConfig +filters = threshold + +filter.threshold.type = ThresholdFilter +filter.threshold.level = ALL + +appender.console.type = Console +appender.console.name = OUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss} %-5p %-10c{1} :: %m%n +#appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss}] %-5p %-10c{1} :: %m%n + +rootLogger.level = INFO +rootLogger.appenderRef.stdout.ref = OUT + +logger.jena.name = org.apache.jena +logger.jena.level = INFO + +logger.arq-exec.name = org.apache.jena.arq.exec +logger.arq-exec.level = INFO + +logger.fuseki.name = org.apache.jena.fuseki +logger.fuseki.level = WARN + +## Some tests correctly log warnings. TS_PrefixesService +logger.fuseki-fuseki.name = org.apache.jena.fuseki.Fuseki +logger.fuseki-fuseki.level = ERROR + +logger.fuseki-autoload.name = org.apache.jena.fuseki.main.sys.FusekiAutoModules +logger.fuseki-autoload.level = ERROR + +logger.http.name = org.apache.jena.http +logger.http.level = INFO + +logger.riot.name = org.apache.jena.riot +logger.riot.level = INFO + +logger.riot.name = org.apache.shiro +logger.riot.level = WARN + +logger.jetty.name = org.eclipse.jetty +logger.jetty.level = WARN + +# This goes out in NCSA format +appender.plain.type = Console +appender.plain.name = PLAIN +appender.plain.layout.type = PatternLayout +appender.plain.layout.pattern = %m%n + +logger.request-log.name = org.apache.jena.fuseki.Request +logger.request-log.additivity = false +logger.request-log.level = OFF +logger.request-log.appenderRef.plain.ref = PLAIN diff --git a/jena-fuseki2/jena-fuseki-server/pom.xml b/jena-fuseki2/jena-fuseki-server/pom.xml index 894c5f6d2b6..222a7ef271e 100644 --- a/jena-fuseki2/jena-fuseki-server/pom.xml +++ b/jena-fuseki2/jena-fuseki-server/pom.xml @@ -66,7 +66,13 @@ jena-cmds ${project.version} - + + + org.apache.jena + jena-fuseki-mod-geosparql + ${project.version} + + org.apache.logging.log4j log4j-slf4j2-impl @@ -95,7 +101,7 @@ false - META-INF/*.SF META-INF/*.DSA diff --git a/jena-fuseki2/pom.xml b/jena-fuseki2/pom.xml index 3dbe63b2123..7adfec9458f 100644 --- a/jena-fuseki2/pom.xml +++ b/jena-fuseki2/pom.xml @@ -57,6 +57,8 @@ jena-fuseki-main + jena-fuseki-mod-geosparql + jena-fuseki-server diff --git a/jena-geosparql/pom.xml b/jena-geosparql/pom.xml index 7c3c80b1e99..e5145a2e937 100644 --- a/jena-geosparql/pom.xml +++ b/jena-geosparql/pom.xml @@ -14,7 +14,9 @@ limitations under the License. --> - + 4.0.0 jena-geosparql jar @@ -33,7 +35,6 @@ - org.apache.jena jena-arq @@ -44,18 +45,6 @@ org.apache.sis.core sis-referencing - - - - - javax.xml.bind - jaxb-api - org.slf4j @@ -78,8 +67,8 @@ - org.apache.commons - commons-collections4 + com.esotericsoftware + kryo @@ -101,7 +90,6 @@ log4j-slf4j2-impl test -
@@ -133,11 +121,11 @@ org.apache.maven.plugins maven-source-plugin - + - attach-sources-test + attach-sources-test - test-jar-no-fork + test-jar-no-fork diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/InitGeoSPARQL.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/InitGeoSPARQL.java index a0ca84639c6..e32482a1a6e 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/InitGeoSPARQL.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/InitGeoSPARQL.java @@ -42,7 +42,7 @@ public static void init() { if ( initialized ) return ; synchronized (initLock) { - if ( initialized ) { + if ( initialized || System.getProperty("jena.geosparql.skip", "false").equalsIgnoreCase("true") ) { JenaSystem.logLifecycle("InitGeoSPARQL - skip") ; return ; } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/GeoAssembler.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/GeoAssembler.java index 5e54e6ad95c..27ecbecc4a4 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/GeoAssembler.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/GeoAssembler.java @@ -18,7 +18,15 @@ package org.apache.jena.geosparql.assembler; -import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.*; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pApplyDefaultGeometry; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pDataset; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexEnabled; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexExpiries; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexSizes; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pInference; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pQueryRewrite; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSpatialIndexFile; +import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSrsUri; import static org.apache.jena.sparql.util.graph.GraphUtils.getBooleanValue; import java.io.IOException; @@ -33,9 +41,8 @@ import org.apache.jena.geosparql.configuration.GeoSPARQLConfig; import org.apache.jena.geosparql.configuration.GeoSPARQLOperations; import org.apache.jena.geosparql.configuration.SrsException; +import org.apache.jena.geosparql.spatial.SpatialIndexConstants; import org.apache.jena.geosparql.spatial.SpatialIndexException; -import org.apache.jena.graph.Graph; -import org.apache.jena.graph.Node; import org.apache.jena.query.Dataset; import org.apache.jena.query.DatasetFactory; import org.apache.jena.rdf.model.Property; @@ -56,8 +63,6 @@ public DatasetGraph createDataset(Assembler a, Resource root) { // Base dataset. DatasetGraph base = super.createBaseDataset(root, pDataset); - Graph graph = root.getModel().getGraph(); - Node subj = root.asNode(); // GeoSPARQL RDFS inference. CLI: names = {"--inference", "-i"} boolean inference = true; @@ -117,9 +122,24 @@ public DatasetGraph createDataset(Assembler a, Resource root) { if (root.hasProperty(pSpatialIndexFile) ) spatialIndexFilename = GraphUtils.getStringValue(root, pSpatialIndexFile); - // ---- Build + if (spatialIndexFilename == null) { + LOG.warn(root + ": No spatial index file location is specified via " + pSpatialIndexFile + ". Spatial index will not be persisted."); + } + + // SRS URI + String srsURI = null; + if (root.hasProperty(pSrsUri) ) { + srsURI = GraphUtils.getStringValue(root, pSrsUri); + } + if (srsURI == null) { + LOG.warn(root + ": No preferred SRS is configured via " + pSrsUri); + } + + + // ---- Build Dataset dataset = DatasetFactory.wrap(base); + dataset.getContext().set(SpatialIndexConstants.symSrsUri, srsURI); // Conversion of data. Startup-only. // needed for w3c:geo/wgs84_pos#lat/log. @@ -142,8 +162,8 @@ public DatasetGraph createDataset(Assembler a, Resource root) { //Setup GeoSPARQL if (indexEnabled) { GeoSPARQLConfig.setupMemoryIndex(indexSizes.get(0), indexSizes.get(1), indexSizes.get(2), - (long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2), - queryRewrite); + (long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2), + queryRewrite); } else { GeoSPARQLConfig.setupNoIndex(queryRewrite); } @@ -166,14 +186,10 @@ private static List getListInteger(Resource r, Property p, int len) { } private static void prepareSpatialExtension(Dataset dataset, String spatialIndex){ - boolean isEmpty = dataset.calculateRead(()->dataset.isEmpty()); - if ( isEmpty && spatialIndex != null ) { - LOG.warn("Dataset empty. Spatial Index not constructed. Server will require restarting after adding data and any updates to build Spatial Index."); - return; + boolean isEmpty = dataset.calculateRead(dataset::isEmpty); + if ( isEmpty ) { + LOG.warn("Dataset is empty. Constructing an empty spatial index that needs to be updated once data is added."); } - if ( isEmpty ) - // Nothing to do. - return; try { // no file given, i.e. in-memory index only @@ -185,23 +201,23 @@ private static void prepareSpatialExtension(Dataset dataset, String spatialIndex // file given but empty -> compute and serialize index Path spatialIndexPath = Path.of(spatialIndex); if ( ! Files.exists(spatialIndexPath) || Files.size(spatialIndexPath) == 0 ) { - GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath.toFile()); + GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath); return; } // load and setup the precomputed index - GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath.toFile()); + GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath); } catch (SrsException ex) { // Data but no spatial data. if ( ! ex.getMessage().startsWith("No SRS found") ) throw ex; - LOG.warn(ex.getMessage()); + LOG.warn(ex.getMessage(), ex); } catch (IOException ex) { IO.exception(ex); return; } catch (SpatialIndexException ex) { String msg = "Failed to create spatial index: "+ex.getMessage(); - LOG.error(msg); + LOG.error(msg, ex); throw new JenaException(msg, ex); } } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/VocabGeoSPARQL.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/VocabGeoSPARQL.java index 49dafdd635b..5156ea2c28d 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/VocabGeoSPARQL.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/VocabGeoSPARQL.java @@ -73,6 +73,10 @@ private static Property property(String shortName) { // "File to load or store the spatial index. Default to " + SPATIAL_INDEX_FILE + " in TDB folder if using TDB and not set. Otherwise spatial index is not stored. public static final Property pSpatialIndexFile = property("spatialIndexFile"); + public static final Property pSpatialIndexPerGraph = property("spatialIndexPerGraph"); + + public static final Property pSrsUri = property("srsUri"); + // Dataset public static final Property pDataset = property("dataset"); } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLConfig.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLConfig.java index 759e5f11f24..dfc5c7aa425 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLConfig.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLConfig.java @@ -17,11 +17,17 @@ */ package org.apache.jena.geosparql.configuration; -import java.io.File; +import java.nio.file.Path; import org.apache.jena.geosparql.geof.topological.RelateFF; import org.apache.jena.geosparql.implementation.datatype.GeometryDatatype; -import org.apache.jena.geosparql.implementation.function_registration.*; +import org.apache.jena.geosparql.implementation.function_registration.Egenhofer; +import org.apache.jena.geosparql.implementation.function_registration.GeometryProperty; +import org.apache.jena.geosparql.implementation.function_registration.NonTopological; +import org.apache.jena.geosparql.implementation.function_registration.RCC8; +import org.apache.jena.geosparql.implementation.function_registration.Relate; +import org.apache.jena.geosparql.implementation.function_registration.SimpleFeatures; +import org.apache.jena.geosparql.implementation.function_registration.Spatial; import org.apache.jena.geosparql.implementation.index.GeometryLiteralIndex; import org.apache.jena.geosparql.implementation.index.GeometryTransformIndex; import org.apache.jena.geosparql.implementation.index.IndexConfiguration; @@ -31,6 +37,9 @@ import org.apache.jena.geosparql.implementation.vocabulary.Geo; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.compat.SpatialIndexIo; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexIoKryo; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; import org.apache.jena.query.Dataset; import org.apache.jena.sparql.function.FunctionRegistry; import org.apache.jena.sparql.pfunction.PropertyFunctionRegistry; @@ -249,7 +258,7 @@ public static final void setupQueryRewriteIndex(Dataset dataset, String queryRew * @throws SpatialIndexException */ public static final void setupSpatialIndex(Dataset dataset) throws SpatialIndexException { - SpatialIndex.buildSpatialIndex(dataset); + SpatialIndexLib.buildSpatialIndex(dataset.asDatasetGraph()); } /** @@ -260,9 +269,9 @@ public static final void setupSpatialIndex(Dataset dataset) throws SpatialIndexE * @param spatialIndexFile the file containing the serialized spatial index * @throws SpatialIndexException */ - public static final void setupPrecomputedSpatialIndex(Dataset dataset, File spatialIndexFile) throws SpatialIndexException { - SpatialIndex si = SpatialIndex.load(spatialIndexFile); - SpatialIndex.setSpatialIndex(dataset, si); + public static final void setupPrecomputedSpatialIndex(Dataset dataset, Path spatialIndexFile) throws SpatialIndexException { + SpatialIndex si = SpatialIndexIo.load(spatialIndexFile); + SpatialIndexLib.setSpatialIndex(dataset, si); } /** @@ -274,8 +283,8 @@ public static final void setupPrecomputedSpatialIndex(Dataset dataset, File spat * @param spatialIndexFile * @throws SpatialIndexException */ - public static final void setupSpatialIndex(Dataset dataset, File spatialIndexFile) throws SpatialIndexException { - SpatialIndex.buildSpatialIndex(dataset, spatialIndexFile); + public static final void setupSpatialIndex(Dataset dataset, Path spatialIndexFile) throws SpatialIndexException { + SpatialIndexIoKryo.loadOrBuildSpatialIndex(dataset, spatialIndexFile); } /** @@ -287,8 +296,8 @@ public static final void setupSpatialIndex(Dataset dataset, File spatialIndexFil * @param spatialIndexFile * @throws SpatialIndexException */ - public static final void setupSpatialIndex(Dataset dataset, String srsURI, File spatialIndexFile) throws SpatialIndexException { - SpatialIndex.buildSpatialIndex(dataset, srsURI, spatialIndexFile); + public static final void setupSpatialIndex(Dataset dataset, String srsURI, Path spatialIndexFile) throws SpatialIndexException { + SpatialIndexIoKryo.loadOrBuildSpatialIndex(dataset, srsURI, spatialIndexFile); } /** diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLOperations.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLOperations.java index 5ecce51d424..4f5bc802f5d 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLOperations.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLOperations.java @@ -17,6 +17,8 @@ */ package org.apache.jena.geosparql.configuration; +import static org.apache.jena.geosparql.configuration.GeoSPARQLConfig.DECIMAL_PLACES_PRECISION; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -32,9 +34,11 @@ import java.util.Set; import java.util.TreeSet; import java.util.UUID; +import java.util.stream.Stream; + +import org.apache.jena.atlas.iterator.Iter; import org.apache.jena.datatypes.DatatypeFormatException; import org.apache.jena.datatypes.RDFDatatype; -import static org.apache.jena.geosparql.configuration.GeoSPARQLConfig.DECIMAL_PLACES_PRECISION; import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.implementation.datatype.GMLDatatype; import org.apache.jena.geosparql.implementation.datatype.GeometryDatatype; @@ -42,8 +46,11 @@ import org.apache.jena.geosparql.implementation.index.GeometryLiteralIndex; import org.apache.jena.geosparql.implementation.vocabulary.Geo; import org.apache.jena.geosparql.implementation.vocabulary.GeoSPARQL_URI; +import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; import org.apache.jena.geosparql.spatial.ConvertLatLon; +import org.apache.jena.geosparql.spatial.SpatialIndexConstants; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; import org.apache.jena.query.Dataset; import org.apache.jena.query.DatasetFactory; import org.apache.jena.query.ReadWrite; @@ -63,6 +70,8 @@ import org.apache.jena.reasoner.ReasonerRegistry; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.system.AutoTxn; +import org.apache.jena.system.Txn; import org.apache.jena.util.iterator.ExtendedIterator; import org.apache.jena.vocabulary.RDFS; import org.opengis.geometry.MismatchedDimensionException; @@ -494,32 +503,58 @@ public static final boolean validateGeometryLiteral(Model model) { return isValid; } + /** + * Return an SRS for a dataset. + * If {@link SpatialIndexConstants#symSrsUri} is set then this SRS is returned. + * Otherwise, {@link #scanModeSRS(Dataset, String)} is called. + */ + public static final String findModeSRS(Dataset dataset) throws SrsException { + // return SRS if set via assembler config + String result = SpatialIndexLib.getPreferredSRS(dataset.getContext()); + + if (result == null) { + LOGGER.info("SRS scan started. This may take a while..."); + + result = scanModeSRS(dataset, null); + + if (result != null) { + LOGGER.info("SRS scan completed. Selecting SRS: " + result); + } else { + result = SRS_URI.DEFAULT_WKT_CRS84; + LOGGER.warn("SRS scan completed. No SRS usage found. Falling back to default SRS: " + result); + } + } + + return result; + } + /** * Find the most frequent SRS URI of Geometry Literals in the dataset. * * @param dataset * @return SRS URI */ - public static final String findModeSRS(Dataset dataset) throws SrsException { - LOGGER.info("Find Mode SRS - Started"); + public static final String scanModeSRS(Dataset dataset, String fallbackSrs) throws SrsException { ModeSRS modeSRS = new ModeSRS(); - //Default Model - dataset.begin(ReadWrite.READ); - Model defaultModel = dataset.getDefaultModel(); - modeSRS.search(defaultModel); + try (AutoTxn txn = Txn.autoTxn(dataset, ReadWrite.READ)) { + //Default Model + Model defaultModel = dataset.getDefaultModel(); + modeSRS.search(defaultModel); - //Named Models - Iterator graphNames = dataset.listNames(); - while (graphNames.hasNext()) { - String graphName = graphNames.next(); - Model namedModel = dataset.getNamedModel(graphName); - modeSRS.search(namedModel); + //Named Models + try (Stream graphNames = Iter.asStream(dataset.listNames())) { + Iterator it = graphNames.iterator(); + while (it.hasNext()) { + String graphName = it.next(); + Model namedModel = dataset.getNamedModel(graphName); + modeSRS.search(namedModel); + } + } + txn.commit(); } - - LOGGER.info("Find Mode SRS - Completed"); - dataset.end(); - - return modeSRS.getModeURI(); + List srsList = modeSRS.getSrsList(); + String result = srsList.isEmpty() ? fallbackSrs : modeSRS.getModeURI(); + return result; } /** @@ -982,24 +1017,24 @@ public static final String convertGeometryLiteral(String geometryLiteral, String /** * Convert the input model to the most frequent coordinate reference system - * and default datatype. + * and default datatype using WKT geometry representation. * * @param inputModel * @return Output of conversion. */ public static final Model convert(Model inputModel) { - return convertSRSDatatype(inputModel, null, null); + return convertSRSDatatype(inputModel, null, WKTDatatype.INSTANCE); } /** - * Convert the input model to the output coordinate reference system. + * Convert the input model to the output coordinate reference system using WKT geometry representation. * * @param inputModel * @param outputSrsURI * @return Output of conversion. */ public static final Model convert(Model inputModel, String outputSrsURI) { - return convertSRSDatatype(inputModel, outputSrsURI, null); + return convertSRSDatatype(inputModel, outputSrsURI, WKTDatatype.INSTANCE); } /** @@ -1041,24 +1076,25 @@ public static final Model convert(Model inputModel, String outputSrsURI, String /** * Convert the input dataset to the most frequent coordinate reference - * system and default datatype. + * system and default datatype using WKT geometry representation. * * @param dataset * @return Converted dataset. */ public static final Dataset convert(Dataset dataset) { - return convert(dataset, null, null); + return convert(dataset, null, WKTDatatype.INSTANCE); } /** - * Convert the input dataset to the output coordinate reference system. + * Convert the input dataset to the output coordinate reference system + * using WKT geometry representation. * * @param dataset * @param outputSrsURI * @return Converted dataset. */ public static final Dataset convert(Dataset dataset, String outputSrsURI) { - return convert(dataset, outputSrsURI, null); + return convert(dataset, outputSrsURI, WKTDatatype.INSTANCE); } /** diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/SrsException.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/SrsException.java index ebc7d5fb163..cc846afffe1 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/SrsException.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/SrsException.java @@ -22,6 +22,7 @@ * */ public class SrsException extends RuntimeException { + private static final long serialVersionUID = 1L; public SrsException(String msg) { super(msg); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunction.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunction.java index e389a681c60..c1185a71add 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunction.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunction.java @@ -18,7 +18,6 @@ package org.apache.jena.geosparql.geo.topological; import java.util.Collection; -import java.util.HashSet; import java.util.List; import org.apache.jena.atlas.iterator.Iter; @@ -29,10 +28,10 @@ import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; import org.apache.jena.graph.Triple; -import org.apache.jena.rdf.model.Resource; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.ExecutionContext; import org.apache.jena.sparql.engine.QueryIterator; @@ -155,7 +154,7 @@ private QueryIterator oneBound(Binding binding, Node subject, Node predicate, No } } - boolean isSpatialIndex = SpatialIndex.isDefined(execCxt); + boolean isSpatialIndex = SpatialIndexLib.isDefined(execCxt); QueryIterator result; if (!isSpatialIndex || filterFunction.isDisjoint() || filterFunction.isDisconnected()) { //Disjointed so retrieve all cases. @@ -230,16 +229,18 @@ private QueryIterator findIndex(Graph graph, Node boundNode, Node unboundNode, B Node geometryLiteral = boundGeometryLiteral.getGeometryLiteral(); - //Perform the search of the Spatial Index of the Dataset. - SpatialIndex spatialIndex = SpatialIndex.retrieve(execCxt); + // Perform the search of the Spatial Index of the Dataset. + SpatialIndex spatialIndex = SpatialIndexLib.retrieve(execCxt); GeometryWrapper geom = GeometryWrapper.extract(geometryLiteral); GeometryWrapper transformedGeom = geom.transform(spatialIndex.getSrsInfo()); + Envelope searchEnvelope = transformedGeom.getEnvelope(); - HashSet features = spatialIndex.query(searchEnvelope); + Node graphName = SpatialIndexLib.unwrapGraphName(graph); + Collection features = spatialIndex.query(searchEnvelope, graphName); // Check each of the Features that match the search. QueryIterator featuresIter = QueryIterPlainWrapper.create( - Iter.map(features.iterator(), feature -> BindingFactory.binding(binding, unboundVar, feature.asNode())), + Iter.map(features.iterator(), feature -> BindingFactory.binding(binding, unboundVar, feature)), execCxt); QueryIterator queryIterator = QueryIter.flatMap(featuresIter, @@ -334,3 +335,4 @@ public Boolean testFilterFunction(Node subjectGeometryLiteral, Node objectGeomet return filterFunction.exec(subjectGeometryLiteral, objectGeometryLiteral); } } + diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteral.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteral.java index fbb8dbdc2dc..42484cec1e1 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteral.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteral.java @@ -27,10 +27,8 @@ import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; import org.apache.jena.graph.NodeFactory; -import org.apache.jena.graph.Triple; import org.apache.jena.system.G; import org.apache.jena.system.RDFDataException; -import org.apache.jena.util.iterator.ExtendedIterator; import org.apache.jena.vocabulary.RDF; /** @@ -126,7 +124,6 @@ protected static final SpatialObjectGeometryLiteral retrieve(Graph graph, Node t if (geometry != null) { //Find the Geometry Literal of the Geometry. - ExtendedIterator iter = graph.find(geometry, Geo.HAS_SERIALIZATION_NODE, null); Node literalNode = G.getSP(graph, geometry, Geo.HAS_SERIALIZATION_NODE); // If hasSerialization not found then check asWKT. if (literalNode == null) diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/DimensionInfo.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/DimensionInfo.java index 6850974af13..073c71a5be1 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/DimensionInfo.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/DimensionInfo.java @@ -17,11 +17,13 @@ */ package org.apache.jena.geosparql.implementation; +import static org.apache.jena.geosparql.implementation.jts.CustomCoordinateSequence.findCoordinateSequenceDimensions; + import java.io.Serializable; import java.util.List; import java.util.Objects; + import org.apache.jena.geosparql.implementation.jts.CoordinateSequenceDimensions; -import static org.apache.jena.geosparql.implementation.jts.CustomCoordinateSequence.findCoordinateSequenceDimensions; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateXY; import org.locationtech.jts.geom.Geometry; @@ -31,6 +33,7 @@ * */ public class DimensionInfo implements Serializable { + private static final long serialVersionUID = 1L; private final int coordinate; private final int spatial; diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/GeometryWrapper.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/GeometryWrapper.java index d93109f20b9..562ccdfaad0 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/GeometryWrapper.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/GeometryWrapper.java @@ -18,8 +18,8 @@ package org.apache.jena.geosparql.implementation; import java.io.Serializable; -import java.lang.invoke.MethodHandles; import java.util.Objects; + import org.apache.jena.datatypes.DatatypeFormatException; import org.apache.jena.geosparql.configuration.GeoSPARQLConfig; import org.apache.jena.geosparql.implementation.datatype.GMLDatatype; @@ -57,16 +57,13 @@ import org.opengis.referencing.operation.MathTransform; import org.opengis.referencing.operation.TransformException; import org.opengis.util.FactoryException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * * */ public class GeometryWrapper implements Serializable { - - private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final long serialVersionUID = 1L; private final DimensionInfo dimensionInfo; private final SRSInfo srsInfo; @@ -449,9 +446,9 @@ public String getUTMZoneURI() throws FactoryException, MismatchedDimensionExcept //Convert to WGS84. Use WGS84 and not CRS84 as assuming WGS8 is more prevalent. CoordinateReferenceSystem wgs84CRS = SRSRegistry.getCRS(SRS_URI.WGS84_CRS); MathTransform transform = MathTransformRegistry.getMathTransform(srsInfo.getCrs(), wgs84CRS); - + DirectPosition wgs84Point = transform.transform(point, null); - + //Find the UTM zone. utmURI = SRSRegistry.findUTMZoneURIFromWGS84(wgs84Point.getOrdinate(0), wgs84Point.getOrdinate(1)); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/SRSInfoException.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/SRSInfoException.java index 59fe658eb11..f3d21079d85 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/SRSInfoException.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/SRSInfoException.java @@ -22,6 +22,7 @@ * */ public class SRSInfoException extends RuntimeException { + private static final long serialVersionUID = 1L; public SRSInfoException(String msg) { super(msg); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsConversionException.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsConversionException.java index 90310bffc90..04f9594029f 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsConversionException.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsConversionException.java @@ -22,6 +22,7 @@ * */ public class UnitsConversionException extends RuntimeException { + private static final long serialVersionUID = 1L; public UnitsConversionException(String msg) { super(msg); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsOfMeasure.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsOfMeasure.java index fab5643227e..d492879592d 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsOfMeasure.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/UnitsOfMeasure.java @@ -17,12 +17,15 @@ */ package org.apache.jena.geosparql.implementation; +import static org.apache.jena.geosparql.configuration.GeoSPARQLOperations.cleanUpPrecision; + import java.io.Serializable; import java.util.Objects; + import javax.measure.Quantity; import javax.measure.Unit; import javax.measure.quantity.Length; -import static org.apache.jena.geosparql.configuration.GeoSPARQLOperations.cleanUpPrecision; + import org.apache.jena.geosparql.implementation.registry.UnitsRegistry; import org.apache.jena.geosparql.implementation.vocabulary.Unit_URI; import org.apache.sis.measure.Quantities; @@ -38,6 +41,7 @@ * */ public class UnitsOfMeasure implements Serializable { + private static final long serialVersionUID = 1L; private final Unit unit; private final String unitURI; diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GMLDatatype.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GMLDatatype.java index 456eb2bc2f2..bc4e9c5d768 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GMLDatatype.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GMLDatatype.java @@ -18,6 +18,7 @@ package org.apache.jena.geosparql.implementation.datatype; import java.io.IOException; + import org.apache.jena.datatypes.DatatypeFormatException; import org.apache.jena.geosparql.implementation.DimensionInfo; import org.apache.jena.geosparql.implementation.GeometryWrapper; @@ -26,8 +27,6 @@ import org.apache.jena.geosparql.implementation.vocabulary.Geo; import org.jdom2.JDOMException; import org.locationtech.jts.geom.Geometry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -35,9 +34,6 @@ * */ public class GMLDatatype extends GeometryDatatype { - - private static final Logger LOGGER = LoggerFactory.getLogger(GMLDatatype.class); - /** * The default GML type URI. */ @@ -96,5 +92,4 @@ public GeometryWrapper read(String geometryLiteral) { public String toString() { return "GMLDatatype{" + URI + '}'; } - } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GeoJSONDatatype.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GeoJSONDatatype.java index 89206a4247c..d1bb25d06a4 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GeoJSONDatatype.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/GeoJSONDatatype.java @@ -19,16 +19,11 @@ import org.apache.jena.datatypes.BaseDatatype; import org.apache.jena.geosparql.implementation.vocabulary.Geo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /* * this GeoJSONDatatype does not yet do anything other than wrap a literal */ public class GeoJSONDatatype extends BaseDatatype { - - private static final Logger LOGGER = LoggerFactory.getLogger(GeoJSONDatatype.class); - /** * The default GML type URI. */ @@ -45,5 +40,4 @@ public class GeoJSONDatatype extends BaseDatatype { private GeoJSONDatatype() { super(URI); } - } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/WKTDatatype.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/WKTDatatype.java index 485ae004034..3f94b8a5415 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/WKTDatatype.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/datatype/WKTDatatype.java @@ -24,8 +24,6 @@ import org.apache.jena.geosparql.implementation.parsers.wkt.WKTWriter; import org.apache.jena.geosparql.implementation.vocabulary.Geo; import org.locationtech.jts.geom.Geometry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * WKTDatatype class allows the URI "geo:wktLiteral" to be used as a datatype @@ -43,9 +41,6 @@ * specify an explicit spatial reference system URI. */ public class WKTDatatype extends GeometryDatatype { - - private static final Logger LOGGER = LoggerFactory.getLogger(WKTDatatype.class); - /** * The default WKT type URI. */ diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequence.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequence.java index 143dfaedaac..5fb8aeee3ee 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequence.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequence.java @@ -17,10 +17,12 @@ */ package org.apache.jena.geosparql.implementation.jts; +import static org.apache.jena.geosparql.implementation.WKTLiteralFactory.reducePrecision; + import java.io.Serializable; import java.util.Arrays; import java.util.List; -import static org.apache.jena.geosparql.implementation.WKTLiteralFactory.reducePrecision; + import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.Envelope; @@ -32,6 +34,7 @@ * */ public class CustomCoordinateSequence implements CoordinateSequence, Serializable { + private static final long serialVersionUID = 1L; private final double[] x; private final double[] y; diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequenceFactory.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequenceFactory.java index 04cddc2d34b..4a787180373 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequenceFactory.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomCoordinateSequenceFactory.java @@ -18,6 +18,7 @@ package org.apache.jena.geosparql.implementation.jts; import java.io.Serializable; + import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.CoordinateSequenceFactory; @@ -27,6 +28,7 @@ * */ public class CustomCoordinateSequenceFactory implements CoordinateSequenceFactory, Serializable { + private static final long serialVersionUID = 1L; @Override public CoordinateSequence create(Coordinate[] coordinates) { diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomGeometryFactory.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomGeometryFactory.java index 61e76dbb13d..1c4a84e9fa9 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomGeometryFactory.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/CustomGeometryFactory.java @@ -17,16 +17,19 @@ */ package org.apache.jena.geosparql.implementation.jts; +import static org.apache.jena.geosparql.configuration.GeoSPARQLConfig.PRECISION_MODEL_SCALE_FACTOR; + import java.io.Serializable; + import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.PrecisionModel; -import static org.apache.jena.geosparql.configuration.GeoSPARQLConfig.PRECISION_MODEL_SCALE_FACTOR; /** * * */ public class CustomGeometryFactory implements Serializable { + private static final long serialVersionUID = 1L; private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(new PrecisionModel(PRECISION_MODEL_SCALE_FACTOR), 0, new CustomCoordinateSequenceFactory()); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/GeometryTransformation.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/GeometryTransformation.java index c8fcfc758b3..7d500734bc3 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/GeometryTransformation.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/jts/GeometryTransformation.java @@ -17,10 +17,11 @@ */ package org.apache.jena.geosparql.implementation.jts; -import java.lang.invoke.MethodHandles; +import static org.apache.jena.geosparql.configuration.GeoSPARQLOperations.cleanUpPrecision; + import java.util.ArrayList; + import org.apache.jena.datatypes.DatatypeFormatException; -import static org.apache.jena.geosparql.configuration.GeoSPARQLOperations.cleanUpPrecision; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.Geometry; @@ -35,8 +36,6 @@ import org.locationtech.jts.geom.Polygon; import org.opengis.referencing.operation.MathTransform; import org.opengis.referencing.operation.TransformException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -44,8 +43,6 @@ */ public class GeometryTransformation { - private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final Geometry transform(Geometry sourceGeometry, MathTransform transform) throws TransformException { String geometryType = sourceGeometry.getGeometryType(); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/MathTransformRegistry.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/MathTransformRegistry.java index c9a92e0be09..3d1498712f1 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/MathTransformRegistry.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/MathTransformRegistry.java @@ -17,7 +17,6 @@ */ package org.apache.jena.geosparql.implementation.registry; -import java.lang.invoke.MethodHandles; import org.apache.commons.collections4.keyvalue.MultiKey; import org.apache.commons.collections4.map.HashedMap; import org.apache.commons.collections4.map.MultiKeyMap; @@ -27,8 +26,6 @@ import org.opengis.referencing.operation.MathTransform; import org.opengis.referencing.operation.TransformException; import org.opengis.util.FactoryException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -36,7 +33,6 @@ */ public class MathTransformRegistry { - private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final MultiKeyMap MATH_TRANSFORM_REGISTRY = MultiKeyMap.multiKeyMap(new HashedMap<>()); public synchronized static final MathTransform getMathTransform(CoordinateReferenceSystem sourceCRS, CoordinateReferenceSystem targetCRS) throws FactoryException, MismatchedDimensionException, TransformException { diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/SRSRegistry.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/SRSRegistry.java index 837290f3ae7..746b294deb3 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/SRSRegistry.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/SRSRegistry.java @@ -17,17 +17,19 @@ */ package org.apache.jena.geosparql.implementation.registry; +import static org.apache.jena.geosparql.implementation.vocabulary.SRS_URI.EPSG_BASE_SRS_URI; + import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.text.DecimalFormat; import java.util.Collections; import java.util.HashMap; import java.util.Map; + import org.apache.jena.geosparql.implementation.SRSInfo; import org.apache.jena.geosparql.implementation.SRSInfoException; import org.apache.jena.geosparql.implementation.UnitsOfMeasure; import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; -import static org.apache.jena.geosparql.implementation.vocabulary.SRS_URI.EPSG_BASE_SRS_URI; import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,7 @@ * */ public class SRSRegistry implements Serializable { + private static final long serialVersionUID = 1L; private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/UnitsURIException.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/UnitsURIException.java index 8bd057e85d4..d49c807258d 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/UnitsURIException.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/implementation/registry/UnitsURIException.java @@ -22,6 +22,7 @@ * */ public class UnitsURIException extends RuntimeException { + private static final long serialVersionUID = 1L; public UnitsURIException(String msg) { super(msg); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/EnvelopeSerializer.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/EnvelopeSerializer.java new file mode 100644 index 00000000000..eaeec35e144 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/EnvelopeSerializer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jena.geosparql.kryo; + +import org.locationtech.jts.geom.Envelope; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +public class EnvelopeSerializer extends Serializer { + @Override + public void write(Kryo kryo, Output output, Envelope envelope) { + output.writeDouble(envelope.getMinX()); + output.writeDouble(envelope.getMaxX()); + output.writeDouble(envelope.getMinY()); + output.writeDouble(envelope.getMaxY()); + } + + @Override + public Envelope read(Kryo kryo, Input input, Class type) { + double xMin = input.readDouble(); + double xMax = input.readDouble(); + double yMin = input.readDouble(); + double yMax = input.readDouble(); + if (xMin <= xMax) { + return new Envelope(xMin, xMax, yMin, yMax); + } else { + // Null envelope cannot be constructed using Envelope(xMin, xMax, yMin, yMax) + return new Envelope(); + } + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/GeometrySerializerJtsWkb.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/GeometrySerializerJtsWkb.java new file mode 100644 index 00000000000..00d66ae7bae --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/GeometrySerializerJtsWkb.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.kryo; + +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +/** Geometry de-/serialization via the WKB facilities of JTS. */ +public class GeometrySerializerJtsWkb + extends Serializer +{ + private transient WKBReader reader; + private transient WKBWriter writer; + + public GeometrySerializerJtsWkb() { + this(new GeometryFactory()); + } + + public GeometrySerializerJtsWkb(GeometryFactory geometryFactory) { + this(new WKBReader(geometryFactory), new WKBWriter()); + } + + public GeometrySerializerJtsWkb(WKBReader reader, WKBWriter writer) { + super(); + this.reader = reader; + this.writer = writer; + } + + @Override + public void write(Kryo kryo, Output output, Geometry geometry) { + byte[] data = writer.write(geometry); + output.write(data, 0, data.length); + kryo.writeClassAndObject(output, geometry.getUserData()); + } + + @Override + public Geometry read(Kryo kryo, Input input, Class type) { + byte[] bytes = kryo.readObject(input, byte[].class); + Geometry geometry; + try { + geometry = reader.read(bytes); + } catch (ParseException e) { + throw new RuntimeException(e); + } + Object userData = kryo.readClassAndObject(input); + geometry.setUserData(userData); + return geometry; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/NodeSerializer.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/NodeSerializer.java new file mode 100644 index 00000000000..74f950d73eb --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/NodeSerializer.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.kryo; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; + +import org.apache.jena.datatypes.RDFDatatype; +import org.apache.jena.datatypes.TypeMapper; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.graph.Node_ANY; +import org.apache.jena.graph.Node_Blank; +import org.apache.jena.graph.Node_Graph; +import org.apache.jena.graph.Node_Literal; +import org.apache.jena.graph.Node_Triple; +import org.apache.jena.graph.Node_URI; +import org.apache.jena.graph.Node_Variable; +import org.apache.jena.graph.TextDirection; +import org.apache.jena.graph.Triple; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.sparql.core.Var; +import org.apache.jena.vocabulary.OWL; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; +import org.apache.jena.vocabulary.XSD; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +/** + * An RDF 1.2 node serializer for kryo. + */ +public class NodeSerializer + extends Serializer +{ + public static void register(Kryo kryo) { + register(kryo, new NodeSerializer()); + } + + /** + * Registers serializers for all Jena Node types - except for Node_Ext and any subclasses. + * + * For Node_Triple and Node_Graph to function, additional serializers for + * Triple and Graph need to be registered. + */ + public static void register(Kryo kryo, Serializer nodeSerializer) { + // Concrete nodes + kryo.register(Node.class, nodeSerializer); + kryo.register(Node_Blank.class, nodeSerializer); + kryo.register(Node_URI.class, nodeSerializer); + kryo.register(Node_Literal.class, nodeSerializer); + kryo.register(Node_Triple.class, nodeSerializer); + + // Variable nodes + kryo.register(Node_Variable.class, nodeSerializer); + kryo.register(Var.class, nodeSerializer); + kryo.register(Node_ANY.class, nodeSerializer); + + // Extensions + kryo.register(Node_Graph.class, nodeSerializer); + } + + protected static final byte TYPE_MASK = (byte)0xf0; // 1111 0000 + protected static final byte TYPE_IRI = 0x10; // 0001 0000 + protected static final byte TYPE_BNODE = 0x20; // 0010 0000 + protected static final byte TYPE_LITERAL = 0x30; // 0011 0000 + protected static final byte TYPE_TRIPLE = 0x40; // 0100 0000 + protected static final byte TYPE_GRAPH = 0x50; // 0101 0000 - Delegates back to kryo + + protected static final byte TYPE_VAR = (byte)0x80; // 1000 0000 - NoteFactory.createVariable + protected static final byte TYPE_SVAR = (byte)0x90; // 1001 0000 - Var.alloc + protected static final byte TYPE_ANY = (byte)0xa0; // 1010 0000 + protected static final byte TYPE_EXT = (byte)0xb0; // 1011 0000 - Delegates back to kryo + + protected static final byte LITERAL_MASK = 0x03; // 0000 0011 + protected static final byte LITERAL_STRING = 0x00; // 0000 0000 + protected static final byte LITERAL_LANG = 0x01; // 0000 0001 + protected static final byte LITERAL_DTYPE = 0x02; // 0000 0010 + + // Iff (TYPE_LITERAL and LITERAL_LANG) the following applies: + protected static final byte LITERAL_LANG_MASK = 0x0c; // 0000 1100 + protected static final byte LITERAL_LANG_LTR = 0x04; // 0000 0100 + protected static final byte LITERAL_LANG_RTL = 0x08; // 0000 1000 + + // Iff (TYPE_IRI) or (TYPE_LITERAL & LITERAL_DYTE): Whether the value field is abbreviated. + protected static final byte ABBREV_IRI = 0x08; // 0000 1000 + + private static Map globalPrefixToIri = new HashMap<>(); + + private Map prefixToIri = new HashMap<>(); + private Map iriToPrefix = new HashMap<>(); + + static { + globalPrefixToIri.put("a", RDF.type.getURI()); + globalPrefixToIri.put("d", Quad.defaultGraphIRI.getURI()); + globalPrefixToIri.put("g", Quad.defaultGraphNodeGenerated.getURI()); + globalPrefixToIri.put("x", XSD.NS); + globalPrefixToIri.put("r", RDF.uri); + globalPrefixToIri.put("s", RDFS.uri); + globalPrefixToIri.put("o", OWL.NS); + } + + /** + * Takes a guess for the namespace URI string to use in abbreviation. + * Finds the part of the IRI string before the last '#', '/', ':' or '.'. + * + * @param iriString String string + * @return String or null + */ + // XXX Should use a trie instead. + private static String getCandidateNamespaceIri(String iriString) { + int n = iriString.length(); + int i; + loop: for (i = n - 1; i >= 0; --i) { + char c = iriString.charAt(i); + switch(c) { + case '#': + case '/': + case ':': + case '.': + break loop; + default: + // continue + } + } + String result = i >= 0 ? iriString.substring(0, i + 1) : null; + return result; + } + + private static String encode(Map iriToPrefix, String iri) { + String result = iriToPrefix.get(iri); + if (result == null) { + String nsIri = getCandidateNamespaceIri(iri); + if (nsIri != null) { + String prefix = iriToPrefix.get(nsIri); + if (prefix != null) { + result = prefix + ":" + iri.substring(nsIri.length()); + } + } + } + return result; + } + + private static String decode(Map prefixToIri, String curie) { + String result; + int idx = curie.indexOf(':'); + if (idx < 0) { + result = prefixToIri.get(curie); + } else { + String prefix = curie.substring(0, idx); + String iri = prefixToIri.get(prefix); + result = iri + curie.substring(idx + 1); + } + return result; + } + + protected TypeMapper typeMapper; + + public NodeSerializer() { + this(TypeMapper.getInstance()); + } + + public NodeSerializer(TypeMapper typeMapper) { + this(typeMapper, globalPrefixToIri); + } + + public NodeSerializer(TypeMapper typeMapper, Map prefixToIri) { + super(); + this.typeMapper = typeMapper; + this.prefixToIri = new HashMap<>(prefixToIri); + + this.iriToPrefix = prefixToIri.entrySet().stream() + .collect(Collectors.toMap(Entry::getValue, Entry::getKey)); + } + + @Override + public void write(Kryo kryo, Output output, Node node) { + if (node.isURI()) { + String uri = node.getURI(); + String curie = encode(iriToPrefix, uri); + if (curie != null) { + output.writeByte(TYPE_IRI | ABBREV_IRI); + output.writeString(curie); + } else { + output.writeByte(TYPE_IRI); + output.writeString(uri); + } + } else if (node.isLiteral()) { + String lex = node.getLiteralLexicalForm(); + String lang = node.getLiteralLanguage(); + String dt = node.getLiteralDatatypeURI(); + TextDirection dir = node.getLiteralBaseDirection(); + + if (lang != null && !lang.isEmpty()) { + byte langDirBits = dir == null ? (byte)0 : switch(dir) { + case LTR -> LITERAL_LANG_LTR; + case RTL -> LITERAL_LANG_RTL; + }; + output.writeByte(TYPE_LITERAL | LITERAL_LANG | langDirBits); + output.writeString(lex); + output.writeString(lang); + } else if (dt != null && !dt.isEmpty() && !dt.equals(XSD.xstring.getURI())) { + String dtCurie = encode(iriToPrefix, dt); + if (dtCurie != null) { + output.writeByte(TYPE_LITERAL | LITERAL_DTYPE | ABBREV_IRI); + output.writeString(lex); + output.writeString(dtCurie); + } else { + output.writeByte(TYPE_LITERAL | LITERAL_DTYPE); + output.writeString(lex); + output.writeString(dt); + } + } else { + output.writeByte(TYPE_LITERAL); + output.writeString(lex); + } + } else if (node.isBlank()) { + output.writeByte(TYPE_BNODE); + output.writeString(node.getBlankNodeLabel()); + } else if (Node.ANY.equals(node)) { + output.writeByte(TYPE_ANY); + } else if (node.isVariable()) { + if (node instanceof Var) { + output.writeByte(TYPE_SVAR); + } else { + output.writeByte(TYPE_VAR); + } + output.writeString(node.getName()); + } else if (node.isTripleTerm()) { + output.writeByte(TYPE_TRIPLE); + kryo.writeObject(output, node.getTriple()); + } else if (node.isNodeGraph()) { + output.writeByte(TYPE_GRAPH); + kryo.writeObject(output, node.getGraph()); + } else if (node.isExt()) { + output.writeByte(TYPE_EXT); + kryo.writeClassAndObject(output, node); + } else { + throw new RuntimeException("Unknown node type: " + node); + } + } + + @Override + public Node read(Kryo kryo, Input input, Class cls) { + Node result; + String v1, v2; + Triple t; + + byte type = input.readByte(); + + int typeVal = type & TYPE_MASK; + switch (typeVal) { + case TYPE_IRI: + v1 = input.readString(); + if ((type & ABBREV_IRI) != 0) { + v1 = decode(prefixToIri, v1); + } + result = NodeFactory.createURI(v1); + break; + case TYPE_LITERAL: + int subTypeVal = type & LITERAL_MASK; + switch (subTypeVal) { + case LITERAL_STRING: + v1 = input.readString(); + result = NodeFactory.createLiteralString(v1); + break; + case LITERAL_LANG: + int langDirBits = type & LITERAL_LANG_MASK; + + TextDirection textDir = switch(langDirBits) { + case LITERAL_LANG_LTR -> TextDirection.LTR; + case LITERAL_LANG_RTL -> TextDirection.RTL; + default -> Node.noTextDirection; + }; + + v1 = input.readString(); + v2 = input.readString(); + result = NodeFactory.createLiteralDirLang(v1, v2, textDir); + break; + case LITERAL_DTYPE: + v1 = input.readString(); + v2 = input.readString(); + if ((type & ABBREV_IRI) != 0) { + v2 = decode(prefixToIri, v2); + } + RDFDatatype dtype = typeMapper.getSafeTypeByName(v2); + result = NodeFactory.createLiteralDT(v1, dtype); + break; + default: + throw new RuntimeException("Unknown literal sub-type: " + subTypeVal); + } + break; + case TYPE_BNODE: + v1 = input.readString(); + result = NodeFactory.createBlankNode(v1); + break; + case TYPE_TRIPLE: + t = kryo.readObject(input, Triple.class); + result = NodeFactory.createTripleTerm(t); + break; + case TYPE_GRAPH: + Graph graph = kryo.readObject(input, Graph.class); + result = new Node_Graph(graph); + break; + case TYPE_ANY: + result = Node.ANY; + break; + case TYPE_SVAR: + v1 = input.readString(); + result = Var.alloc(v1); + break; + case TYPE_VAR: + v1 = input.readString(); + result = NodeFactory.createVariable(v1); + break; + case TYPE_EXT: + Object o = kryo.readClassAndObject(input); + result = (Node)o; + break; + default: + throw new RuntimeException("Unknown node type: " + typeVal); + } + return result; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/TripleSerializer.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/TripleSerializer.java new file mode 100644 index 00000000000..d4582211d86 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/kryo/TripleSerializer.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.kryo; + +import org.apache.jena.graph.Node; +import org.apache.jena.graph.Triple; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +/** Kryo serializer for {@link Triple}. Depends on registered {@link Node} serializers. */ +public class TripleSerializer extends Serializer { + @Override + public void write(Kryo kryo, Output output, Triple obj) { + kryo.writeClassAndObject(output, obj.getSubject()); + kryo.writeClassAndObject(output, obj.getPredicate()); + kryo.writeClassAndObject(output, obj.getObject()); + } + + @Override + public Triple read(Kryo kryo, Input input, Class objClass) { + Node s = (Node)kryo.readClassAndObject(input); + Node p = (Node)kryo.readClassAndObject(input); + Node o = (Node)kryo.readClassAndObject(input); + Triple result = Triple.create(s, p, o); + return result; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SearchEnvelope.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SearchEnvelope.java index 7ab0ea98796..561a48ebbc1 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SearchEnvelope.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SearchEnvelope.java @@ -18,14 +18,16 @@ package org.apache.jena.geosparql.spatial; import java.lang.invoke.MethodHandles; -import java.util.HashSet; +import java.util.Collection; import java.util.Objects; + import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.implementation.SRSInfo; import org.apache.jena.geosparql.implementation.UnitsOfMeasure; import org.apache.jena.geosparql.implementation.great_circle.GreatCirclePointDistance; import org.apache.jena.geosparql.implementation.great_circle.LatLonPoint; -import org.apache.jena.rdf.model.Resource; +import org.apache.jena.graph.Node; +import org.apache.jena.sparql.core.Quad; import org.apache.jena.sparql.expr.ExprEvalException; import org.locationtech.jts.geom.Envelope; import org.opengis.geometry.MismatchedDimensionException; @@ -107,11 +109,28 @@ public SRSInfo getCrsInfo() { return srsInfo; } - public HashSet check(SpatialIndex spatialIndex) { - HashSet features = spatialIndex.query(mainEnvelope); + +// public Collection check(SpatialIndex spatialIndex) { +// Collection features = spatialIndex.query(mainEnvelope); +// +// if (wrapEnvelope != null) { +// Collection wrapFeatures = spatialIndex.query(wrapEnvelope); +// features.addAll(wrapFeatures); +// } +// return features; +// } + + // Check default graph only + public Collection check(SpatialIndex spatialIndex) { + return check(spatialIndex, Quad.defaultGraphIRI); + } + + // Check within a single graph; null for default graph. + public Collection check(SpatialIndex spatialIndex, Node graph) { + Collection features = spatialIndex.query(mainEnvelope, graph); if (wrapEnvelope != null) { - HashSet wrapFeatures = spatialIndex.query(wrapEnvelope); + Collection wrapFeatures = spatialIndex.query(wrapEnvelope, graph); features.addAll(wrapFeatures); } return features; diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndex.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndex.java index 63f48f244f7..3fb8eb6a1e2 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndex.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndex.java @@ -17,530 +17,45 @@ */ package org.apache.jena.geosparql.spatial; -import java.io.*; -import java.lang.invoke.MethodHandles; -import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; +import java.util.Collection; -import org.apache.jena.atlas.RuntimeIOException; -import org.apache.jena.atlas.io.IOX; -import org.apache.jena.geosparql.configuration.GeoSPARQLOperations; -import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.implementation.SRSInfo; -import org.apache.jena.geosparql.implementation.registry.SRSRegistry; -import org.apache.jena.geosparql.implementation.vocabulary.Geo; -import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; -import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; -import org.apache.jena.query.Dataset; -import org.apache.jena.query.DatasetFactory; -import org.apache.jena.query.ReadWrite; -import org.apache.jena.rdf.model.*; -import org.apache.jena.sparql.engine.ExecutionContext; -import org.apache.jena.sparql.util.Context; -import org.apache.jena.sparql.util.Symbol; -import org.apache.jena.util.iterator.ExtendedIterator; +import org.apache.jena.graph.Node; +import org.apache.jena.sparql.core.Quad; import org.locationtech.jts.geom.Envelope; -import org.locationtech.jts.index.strtree.STRtree; -import org.opengis.geometry.MismatchedDimensionException; -import org.opengis.referencing.operation.TransformException; -import org.opengis.util.FactoryException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** - * SpatialIndex for testing bounding box collisions between geometries within a - * Dataset.
- * Queries must be performed using the same SRS URI as the SpatialIndex.
- * The SpatialIndex is added to the Dataset Context when it is built.
- * QueryRewriteIndex is also stored in the SpatialIndex as its content is - * Dataset specific. - * - */ -public class SpatialIndex { - - private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final Symbol SPATIAL_INDEX_SYMBOL = Symbol.create("http://jena.apache.org/spatial#index"); - - private transient final SRSInfo srsInfo; - private boolean isBuilt; - private final STRtree strTree; - private static final int MINIMUM_CAPACITY = 2; - - private SpatialIndex() { - this.strTree = new STRtree(MINIMUM_CAPACITY); - this.isBuilt = true; - this.strTree.build(); - this.srsInfo = SRSRegistry.getSRSInfo(SRS_URI.DEFAULT_WKT_CRS84); - } - - /** - * Unbuilt Spatial Index with provided capacity. - * - * @param capacity - * @param srsURI - */ - public SpatialIndex(int capacity, String srsURI) { - int indexCapacity = capacity < MINIMUM_CAPACITY ? MINIMUM_CAPACITY : capacity; - this.strTree = new STRtree(indexCapacity); - this.isBuilt = false; - this.srsInfo = SRSRegistry.getSRSInfo(srsURI); - } - - /** - * Built Spatial Index with provided capacity. - * - * @param spatialIndexItems - * @param srsURI - * @throws SpatialIndexException - */ - public SpatialIndex(Collection spatialIndexItems, String srsURI) throws SpatialIndexException { - int indexCapacity = spatialIndexItems.size() < MINIMUM_CAPACITY ? MINIMUM_CAPACITY : spatialIndexItems.size(); - this.strTree = new STRtree(indexCapacity); - insertItems(spatialIndexItems); - this.strTree.build(); - this.isBuilt = true; - this.srsInfo = SRSRegistry.getSRSInfo(srsURI); - } +public interface SpatialIndex { /** * * @return Information about the SRS used by the SpatialIndex. */ - public SRSInfo getSrsInfo() { - return srsInfo; - } + SRSInfo getSrsInfo(); /** * * @return True if the SpatialIndex is empty. */ - public boolean isEmpty() { - return strTree.isEmpty(); - } - - /** - * - * @return True if the SpatialIndex has been built. - */ - public boolean isBuilt() { - return isBuilt; - } - - /** - * Build the Spatial Index. No more items can be added. - */ - public void build() { - if (!isBuilt) { - strTree.build(); - isBuilt = true; - } - } + boolean isEmpty(); /** - * Items to add to an unbuilt Spatial Index. - * - * @param indexItems - * @throws SpatialIndexException + * Returns the number of items in the index. */ - public final void insertItems(Collection indexItems) throws SpatialIndexException { - - for (SpatialIndexItem indexItem : indexItems) { - insertItem(indexItem.getEnvelope(), indexItem.getItem()); - } - } + long getSize(); /** - * Item to add to an unbuilt Spatial Index. + * Query the index for all matching items in the given graph. * - * @param envelope - * @param item - * @throws SpatialIndexException - */ - public final void insertItem(Envelope envelope, Resource item) throws SpatialIndexException { - if (!isBuilt) { - strTree.insert(envelope, item); - } else { - throw new SpatialIndexException("SpatialIndex has been built and cannot have additional items."); - } - } - - @SuppressWarnings("unchecked") - public HashSet query(Envelope searchEnvelope) { - if (!strTree.isEmpty()) { - return new HashSet<>(strTree.query(searchEnvelope)); - } else { - return new HashSet<>(); - } - } - - @Override - public String toString() { - return "SpatialIndex{" + "srsInfo=" + srsInfo + ", isBuilt=" + isBuilt + ", strTree=" + strTree + '}'; - } - - /** - * Retrieve the SpatialIndex from the Context. - * - * @param execCxt - * @return SpatialIndex contained in the Context. - * @throws SpatialIndexException - */ - public static final SpatialIndex retrieve(ExecutionContext execCxt) throws SpatialIndexException { - - Context context = execCxt.getContext(); - SpatialIndex spatialIndex = (SpatialIndex) context.get(SPATIAL_INDEX_SYMBOL, null); - - if (spatialIndex == null) { - throw new SpatialIndexException("Dataset Context does not contain SpatialIndex."); - } - - return spatialIndex; - } - - /** - * - * @param execCxt - * @return True if a SpatialIndex is defined in the ExecutionContext. - */ - public static final boolean isDefined(ExecutionContext execCxt) { - Context context = execCxt.getContext(); - return context.isDefined(SPATIAL_INDEX_SYMBOL); - } - - /** - * Set the SpatialIndex into the Context of the Dataset for later retrieval - * and use in spatial functions. - * - * @param dataset - * @param spatialIndex - */ - public static final void setSpatialIndex(Dataset dataset, SpatialIndex spatialIndex) { - Context context = dataset.getContext(); - context.set(SPATIAL_INDEX_SYMBOL, spatialIndex); - } - - /** - * Build Spatial Index from all graphs in Dataset.
- * Dataset contains SpatialIndex in Context.
- * Spatial Index written to file. - * - * @param dataset - * @param srsURI - * @param spatialIndexFile - * @return SpatialIndex constructed. - * @throws SpatialIndexException - */ - public static SpatialIndex buildSpatialIndex(Dataset dataset, String srsURI, File spatialIndexFile) throws SpatialIndexException { - - SpatialIndex spatialIndex = load(spatialIndexFile); - - if (spatialIndex.isEmpty()) { - Collection spatialIndexItems = findSpatialIndexItems(dataset, srsURI); - save(spatialIndexFile, spatialIndexItems, srsURI); - spatialIndex = new SpatialIndex(spatialIndexItems, srsURI); - spatialIndex.build(); - } - - setSpatialIndex(dataset, spatialIndex); - return spatialIndex; - } - - /** - * Build Spatial Index from all graphs in Dataset.
- * Dataset contains SpatialIndex in Context.
- * SRS URI based on most frequent found in Dataset.
- * Spatial Index written to file. - * - * @param dataset - * @param spatialIndexFile - * @return SpatialIndex constructed. - * @throws SpatialIndexException - */ - public static SpatialIndex buildSpatialIndex(Dataset dataset, File spatialIndexFile) throws SpatialIndexException { - String srsURI = GeoSPARQLOperations.findModeSRS(dataset); - SpatialIndex spatialIndex = buildSpatialIndex(dataset, srsURI, spatialIndexFile); - return spatialIndex; - } - - /** - * Build Spatial Index from all graphs in Dataset.
- * Dataset contains SpatialIndex in Context. - * - * @param dataset - * @param srsURI - * @return SpatialIndex constructed. - * @throws SpatialIndexException - */ - public static SpatialIndex buildSpatialIndex(Dataset dataset, String srsURI) throws SpatialIndexException { - LOGGER.info("Building Spatial Index - Started"); - - Collection items = findSpatialIndexItems(dataset, srsURI); - SpatialIndex spatialIndex = new SpatialIndex(items, srsURI); - spatialIndex.build(); - setSpatialIndex(dataset, spatialIndex); - LOGGER.info("Building Spatial Index - Completed"); - return spatialIndex; - } - - /** - * Find Spatial Index Items from all graphs in Dataset.
- * - * @param dataset - * @param srsURI - * @return SpatialIndexItems found. - * @throws SpatialIndexException - */ - public static Collection findSpatialIndexItems(Dataset dataset, String srsURI) throws SpatialIndexException { - //Default Model - dataset.begin(ReadWrite.READ); - Model defaultModel = dataset.getDefaultModel(); - Collection items = getSpatialIndexItems(defaultModel, srsURI); - - //Named Models - Iterator graphNames = dataset.listNames(); - while (graphNames.hasNext()) { - String graphName = graphNames.next(); - Model namedModel = dataset.getNamedModel(graphName); - Collection graphItems = getSpatialIndexItems(namedModel, srsURI); - items.addAll(graphItems); - } - - dataset.end(); - - return items; - } - - /** - * Build Spatial Index from all graphs in Dataset.
- * Dataset contains SpatialIndex in Context.
- * SRS URI based on most frequent found in Dataset. - * - * @param dataset - * @return SpatialIndex constructed. - * @throws SpatialIndexException - */ - public static SpatialIndex buildSpatialIndex(Dataset dataset) throws SpatialIndexException { - String srsURI = GeoSPARQLOperations.findModeSRS(dataset); - SpatialIndex spatialIndex = buildSpatialIndex(dataset, srsURI); - return spatialIndex; - } - - /** - * Wrap Model in a Dataset and build SpatialIndex. - * - * @param model - * @param srsURI - * @return Dataset with default Model and SpatialIndex in Context. - * @throws SpatialIndexException - */ - public static final Dataset wrapModel(Model model, String srsURI) throws SpatialIndexException { - - Dataset dataset = DatasetFactory.createTxnMem(); - dataset.setDefaultModel(model); - buildSpatialIndex(dataset, srsURI); - - return dataset; - } - - /** - * Wrap Model in a Dataset and build SpatialIndex. - * - * @param model - * @return Dataset with default Model and SpatialIndex in Context. - * @throws SpatialIndexException - */ - public static final Dataset wrapModel(Model model) throws SpatialIndexException { - Dataset dataset = DatasetFactory.createTxnMem(); - dataset.setDefaultModel(model); - String srsURI = GeoSPARQLOperations.findModeSRS(dataset); - buildSpatialIndex(dataset, srsURI); - - return dataset; - } - - /** - * Find items from the Model transformed to the SRS URI. - * - * @param model - * @param srsURI - * @return Items found in the Model in the SRS URI. - * @throws SpatialIndexException - */ - public static final Collection getSpatialIndexItems(Model model, String srsURI) throws SpatialIndexException { - - List items = new ArrayList<>(); - - //Only add one set of statements as a converted dataset will duplicate the same info. - if (model.contains(null, Geo.HAS_GEOMETRY_PROP, (Resource) null)) { - LOGGER.info("Feature-hasGeometry-Geometry statements found."); - if (model.contains(null, SpatialExtension.GEO_LAT_PROP, (Literal) null)) { - LOGGER.warn("Lat/Lon Geo predicates also found but will not be added to index."); - } - Collection geometryLiteralItems = getGeometryLiteralIndexItems(model, srsURI); - items.addAll(geometryLiteralItems); - } else if (model.contains(null, SpatialExtension.GEO_LAT_PROP, (Literal) null)) { - LOGGER.info("Geo predicate statements found."); - Collection geoPredicateItems = getGeoPredicateIndexItems(model, srsURI); - items.addAll(geoPredicateItems); - } - - return items; - } - - /** - * - * @param model - * @param srsURI - * @return GeometryLiteral items prepared for adding to SpatialIndex. - * @throws SpatialIndexException - */ - private static Collection getGeometryLiteralIndexItems(Model model, String srsURI) throws SpatialIndexException { - List items = new ArrayList<>(); - StmtIterator stmtIt = model.listStatements(null, Geo.HAS_GEOMETRY_PROP, (Resource) null); - while (stmtIt.hasNext()) { - Statement stmt = stmtIt.nextStatement(); - - Resource feature = stmt.getSubject(); - Resource geometry = stmt.getResource(); - - ExtendedIterator nodeIter = model.listObjectsOfProperty(geometry, Geo.HAS_SERIALIZATION_PROP); - if (!nodeIter.hasNext()) { - NodeIterator wktNodeIter = model.listObjectsOfProperty(geometry, Geo.AS_WKT_PROP); - NodeIterator gmlNodeIter = model.listObjectsOfProperty(geometry, Geo.AS_GML_PROP); - nodeIter = wktNodeIter.andThen(gmlNodeIter); - } - - while (nodeIter.hasNext()) { - Literal geometryLiteral = nodeIter.next().asLiteral(); - GeometryWrapper geometryWrapper = GeometryWrapper.extract(geometryLiteral); - - try { - //Ensure all entries in the target SRS URI. - GeometryWrapper transformedGeometryWrapper = geometryWrapper.convertSRS(srsURI); - - Envelope envelope = transformedGeometryWrapper.getEnvelope(); - SpatialIndexItem item = new SpatialIndexItem(envelope, feature); - items.add(item); - } catch (FactoryException | MismatchedDimensionException | TransformException ex) { - throw new SpatialIndexException("Transformation Exception: " + geometryLiteral + ". " + ex.getMessage()); - } - - } - } - return items; - } - - /** - * - * @param model - * @param srsURI - * @return Geo predicate objects prepared for adding to SpatialIndex. - */ - private static Collection getGeoPredicateIndexItems(Model model, String srsURI) throws SpatialIndexException { - List items = new ArrayList<>(); - ResIterator resIt = model.listResourcesWithProperty(SpatialExtension.GEO_LAT_PROP); - - while (resIt.hasNext()) { - Resource feature = resIt.nextResource(); - - Literal lat = feature.getRequiredProperty(SpatialExtension.GEO_LAT_PROP).getLiteral(); - Literal lon = feature.getProperty(SpatialExtension.GEO_LON_PROP).getLiteral(); - if (lon == null) { - LOGGER.warn("Geo predicates: latitude found but not longitude. " + feature); - continue; - } - - Literal latLonPoint = ConvertLatLon.toLiteral(lat.getFloat(), lon.getFloat()); - GeometryWrapper geometryWrapper = GeometryWrapper.extract(latLonPoint); - - try { - //Ensure all entries in the target SRS URI. - GeometryWrapper transformedGeometryWrapper = geometryWrapper.convertSRS(srsURI); - - Envelope envelope = transformedGeometryWrapper.getEnvelope(); - SpatialIndexItem item = new SpatialIndexItem(envelope, feature); - items.add(item); - } catch (FactoryException | MismatchedDimensionException | TransformException ex) { - throw new SpatialIndexException("Transformation Exception: " + geometryWrapper.getLexicalForm() + ". " + ex.getMessage()); - } - } - return items; - } - - /** - * Load a SpatialIndex from file.
- * Index will be built and empty if file does not exist or is null. - * - * @param spatialIndexFile - * @return Built Spatial Index. - * @throws SpatialIndexException - */ - public static final SpatialIndex load(File spatialIndexFile) throws SpatialIndexException { - - if (spatialIndexFile != null && spatialIndexFile.exists()) { - LOGGER.info("Loading Spatial Index - Started: {}", spatialIndexFile.getAbsolutePath()); - //Cannot directly store the SpatialIndex due to Resources not being serializable, use SpatialIndexStorage class. - try (ObjectInputStream in = new ObjectInputStream(new FileInputStream(spatialIndexFile))) { - SpatialIndexStorage storage = (SpatialIndexStorage) in.readObject(); - - SpatialIndex spatialIndex = storage.getSpatialIndex(); - LOGGER.info("Loading Spatial Index - Completed: {}", spatialIndexFile.getAbsolutePath()); - return spatialIndex; - } catch (ClassNotFoundException | IOException ex) { - throw new SpatialIndexException("Loading Exception: " + ex.getMessage(), ex); - } - } else { - return new SpatialIndex(); - } - } - - /** - * Save SpatialIndex contents to file. - * - * @param spatialIndexFileURI - * @param spatialIndexItems - * @param srsURI - * @throws SpatialIndexException - */ - public static final void save(String spatialIndexFileURI, Collection spatialIndexItems, String srsURI) throws SpatialIndexException { - save(new File(spatialIndexFileURI), spatialIndexItems, srsURI); - } - - /** - * Save SpatialIndex contents to file. + * The default graph can be referenced with null or any value for which {@link Quad#isDefaultGraph()} returns true. + * The union graph (= all named graphs but not the default graph) can be referenced with {@link Quad#unionGraph}. * - * @param spatialIndexFile - * @param spatialIndexItems - * @param srsURI - * @throws SpatialIndexException + * @param searchEnvelope The bounding box for which to find overlapping items. + * @param graph The graph to which to restrict the query. + * @return The collection of nodes whose envelopes overlap with the search envelope. */ - public static final void save(File spatialIndexFile, Collection spatialIndexItems, String srsURI) throws SpatialIndexException { + Collection query(Envelope searchEnvelope, Node graph); - //Cannot directly store the SpatialIndex due to Resources not being serializable, use SpatialIndexStorage class. - if (spatialIndexFile != null) { - LOGGER.info("Saving Spatial Index - Started: {}", spatialIndexFile.getAbsolutePath()); - SpatialIndexStorage storage = new SpatialIndexStorage(spatialIndexItems, srsURI); - String filename = spatialIndexFile.getAbsolutePath(); - Path file = Path.of(filename); - Path tmpFile = IOX.uniqueDerivedPath(file, null); - try { - Files.deleteIfExists(file); - } catch (IOException ex) { - throw new SpatialIndexException("Failed to delete file: " + ex.getMessage()); - } - try { - IOX.safeWriteOrCopy(file, tmpFile, - out->{ - ObjectOutputStream oos = new ObjectOutputStream(out); - oos.writeObject(storage); - }); - } catch (RuntimeIOException ex) { - throw new SpatialIndexException("Save Exception: " + ex.getMessage()); - } finally { - LOGGER.info("Saving Spatial Index - Completed: {}", spatialIndexFile.getAbsolutePath()); - } - } - } + Path getLocation(); + void setLocation(Path location); } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexConstants.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexConstants.java new file mode 100644 index 00000000000..f9f9490d90e --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexConstants.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.geosparql.spatial; + +import org.apache.jena.sparql.util.Symbol; + +public class SpatialIndexConstants { + + public static final Symbol symSpatialIndex = Symbol.create("http://jena.apache.org/spatial#index"); + public static final Symbol symSrsUri = Symbol.create("http://jena.apache.org/spatial#srsURI"); + + /** Symbol for a running task in a dataset's context. Used to synchronize tasks that modify the spatial index. */ + public static final Symbol symSpatialIndexTask = Symbol.create("http://jena.apache.org/spatial#indexTask"); +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexException.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexException.java index a50bd949fbd..6928e3a0996 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexException.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexException.java @@ -22,6 +22,7 @@ * */ public class SpatialIndexException extends Exception { + private static final long serialVersionUID = 1L; public SpatialIndexException(String msg) { super(msg); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexFindUtils.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexFindUtils.java new file mode 100644 index 00000000000..cc9efa7e42e --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexFindUtils.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial; + +import java.lang.invoke.MethodHandles; +import java.util.Iterator; + +import org.apache.jena.atlas.iterator.Iter; +import org.apache.jena.atlas.iterator.IteratorCloseable; +import org.apache.jena.geosparql.implementation.GeometryWrapper; +import org.apache.jena.geosparql.implementation.vocabulary.Geo; +import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.Triple; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.system.G; +import org.locationtech.jts.geom.Envelope; +import org.opengis.geometry.MismatchedDimensionException; +import org.opengis.referencing.operation.TransformException; +import org.opengis.util.FactoryException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SpatialIndexFindUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Find Spatial Index Items from all graphs in Dataset.
+ * + * @param datasetGraph + * @param srsURI + * @return SpatialIndexItems found. + */ + public static IteratorCloseable findSpatialIndexItems(DatasetGraph datasetGraph, String srsURI) { + Graph defaultGraph = datasetGraph.getDefaultGraph(); + IteratorCloseable itemsIter = findSpatialIndexItems(defaultGraph, srsURI); + try { + //Named Models + Iterator graphNodeIt = datasetGraph.listGraphNodes(); + Iterator namedGraphItemsIt = Iter.iter(graphNodeIt).flatMap(graphNode -> { + Graph namedGraph = datasetGraph.getGraph(graphNode); + IteratorCloseable graphItems = findSpatialIndexItems(namedGraph, srsURI); + return graphItems; + }); + itemsIter = Iter.iter(itemsIter).append(namedGraphItemsIt); + } catch(Throwable t) { + t.addSuppressed(new RuntimeException("Failure during findSpatialIndexItems.", t)); + Iter.close(itemsIter); + throw t; + } + return itemsIter; + } + + /** + * Find items from the Model transformed to the SRS URI. + * + * @param graph + * @param srsURI + * @return Items found in the Model in the SRS URI. + */ + public static final IteratorCloseable findSpatialIndexItems(Graph graph, String srsURI) { + IteratorCloseable result; + // Only add one set of statements as a converted dataset will duplicate the same info. + if (graph.contains(null, Geo.HAS_GEOMETRY_NODE, null)) { + // LOGGER.info("Feature-hasGeometry-Geometry statements found."); + // if (graph.contains(null, SpatialExtension.GEO_LAT_NODE, null)) { + // LOGGER.warn("Lat/Lon Geo predicates also found but will not be added to index."); + // } + result = findGeometryIndexItems(graph, srsURI); + } else if (graph.contains(null, SpatialExtension.GEO_LAT_NODE, null)) { + // LOGGER.info("Geo predicate statements found."); + result = findGeoPredicateIndexItems(graph, srsURI); + } else { + result = Iter.empty(); + } + return result; + } + + /** Print out log messages for what type of spatial data is found in the given graph. */ + public static final void checkSpatialIndexItems(Graph graph) { + // Only add one set of statements as a converted dataset will duplicate the same info. + if (graph.contains(null, Geo.HAS_GEOMETRY_NODE, null)) { + LOGGER.info("Feature-hasGeometry-Geometry statements found."); + if (graph.contains(null, SpatialExtension.GEO_LAT_NODE, null)) { + LOGGER.warn("Lat/Lon Geo predicates also found but will not be added to index."); + } + } else if (graph.contains(null, SpatialExtension.GEO_LAT_NODE, null)) { + LOGGER.info("Geo predicate statements found."); + } + } + + /** + * + * @param graph + * @param srsURI + * @return SpatialIndexItem items prepared for adding to SpatialIndex. + */ + public static IteratorCloseable findGeometryIndexItems(Graph graph, String srsURI) { + Iterator stmtIter = graph.find(null, Geo.HAS_GEOMETRY_NODE, null); + IteratorCloseable result = Iter.iter(stmtIter).flatMap(stmt -> { + Node feature = stmt.getSubject(); + Node geometry = stmt.getObject(); + + Iterator nodeIter = G.iterSP(graph, geometry, Geo.HAS_SERIALIZATION_NODE); + + // XXX If there is a super-property then the concrete serializations are not tried. + try { + if (!nodeIter.hasNext()) { + Iter.close(nodeIter); + + Iterator wktNodeIter = G.iterSP(graph, geometry, Geo.AS_WKT_NODE); + nodeIter = wktNodeIter; + + Iterator gmlNodeIter = G.iterSP(graph, geometry, Geo.AS_GML_NODE); + nodeIter = Iter.append(wktNodeIter, gmlNodeIter); + } + } catch (Throwable t) { + t.addSuppressed(new RuntimeException("Error encountered.", t)); + Iter.close(nodeIter); + throw t; + } + + Iterator itemIter = Iter.map(nodeIter, geometryNode -> { + GeometryWrapper geometryWrapper = GeometryWrapper.extract(geometryNode); + SpatialIndexItem item = makeSpatialIndexItem(feature, geometryWrapper, srsURI); + return item; + }); + return itemIter; + }); + return result; + } + + /** + * + * @param graph + * @param srsURI + * @return Geo predicate objects prepared for adding to SpatialIndex. + */ + public static IteratorCloseable findGeoPredicateIndexItems(Graph graph, String srsURI) { + // Warn about multiple lat/lon combinations only at most once per graph. + boolean enableWarnings = false; + boolean[] loggedMultipleLatLons = { false }; + Iterator latIt = graph.find(Node.ANY, SpatialExtension.GEO_LAT_NODE, Node.ANY); + IteratorCloseable result = Iter.iter(latIt).flatMap(triple -> { + Node feature = triple.getSubject(); + Node lat = triple.getObject(); + + // Create the cross-product between lats and lons. + Iterator lons = G.iterSP(graph, feature, SpatialExtension.GEO_LON_NODE); + + // On malformed data this can cause lots of log output. Perhaps it's better to keep validation separate from indexing. + int[] lonCounter = {0}; + Iterator r = Iter.iter(lons).map(lon -> { + if (enableWarnings) { + if (lonCounter[0] == 1) { + if (!loggedMultipleLatLons[0]) { + LOGGER.warn("Geo predicates: multiple longitudes detected on feature " + feature + ". Further warnings will be omitted."); + loggedMultipleLatLons[0] = true; + } + } + ++lonCounter[0]; + } + GeometryWrapper geometryWrapper = ConvertLatLon.toGeometryWrapper(lat, lon); + SpatialIndexItem item = makeSpatialIndexItem(feature, geometryWrapper, srsURI); + return item; + }); + return r; + }); + return result; + } + + public static SpatialIndexItem makeSpatialIndexItem(Node feature, GeometryWrapper geometryWrapper, String srsURI) { + // Ensure all entries in the target SRS URI. + GeometryWrapper transformedGeometryWrapper = unsafeConvert(geometryWrapper, srsURI); + Envelope envelope = transformedGeometryWrapper.getEnvelope(); + SpatialIndexItem item = new SpatialIndexItem(envelope, feature); + return item; + } + + public static GeometryWrapper unsafeConvert(GeometryWrapper geometryWrapper, String srsURI) { + GeometryWrapper result; + try { + result = geometryWrapper.convertSRS(srsURI); + } catch (MismatchedDimensionException | FactoryException | TransformException e) { + throw new RuntimeException(e); + } + return result; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexItem.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexItem.java index 8e6e7a03866..1abff73da3a 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexItem.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexItem.java @@ -17,7 +17,8 @@ */ package org.apache.jena.geosparql.spatial; -import org.apache.jena.rdf.model.Resource; +import org.apache.jena.graph.Node; +import org.apache.jena.rdf.model.RDFNode; import org.locationtech.jts.geom.Envelope; /** @@ -27,9 +28,23 @@ public class SpatialIndexItem { private final Envelope envelope; - private final Resource item; + private final Node item; - public SpatialIndexItem(Envelope envelope, Resource item) { + @Deprecated // (forRemoval = true) + public static SpatialIndexItem of(Envelope envelope, RDFNode item) { + return new SpatialIndexItem(envelope, item); + } + + public static SpatialIndexItem of(Envelope envelope, Node node) { + return new SpatialIndexItem(envelope, node); + } + + @Deprecated // (forRemoval = true) + public SpatialIndexItem(Envelope envelope, RDFNode item) { + this(envelope, item.asNode()); + } + + public SpatialIndexItem(Envelope envelope, Node item) { this.envelope = envelope; this.item = item; } @@ -38,7 +53,7 @@ public Envelope getEnvelope() { return envelope; } - public Resource getItem() { + public Node getItem() { return item; } @@ -46,5 +61,4 @@ public Resource getItem() { public String toString() { return "SpatialIndexItem{" + "envelope=" + envelope + ", item=" + item + '}'; } - } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexStorage.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexStorage.java index 410e8b4b34b..dacaba1915f 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexStorage.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/SpatialIndexStorage.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; + +import org.apache.jena.geosparql.spatial.index.v1.SpatialIndexV1; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.ResourceFactory; import org.locationtech.jts.geom.Envelope; @@ -29,6 +31,7 @@ * Spatial Index Items in a Serializable form for file reading or writing. * */ +@Deprecated /** Serializable java class of spatial index v1. Moving this class would break Java serialization. */ public class SpatialIndexStorage implements Serializable { private final String srsURI; @@ -41,7 +44,7 @@ public SpatialIndexStorage(Collection spatialIndexItems, Strin this.storageItems = new ArrayList<>(spatialIndexItems.size()); for (SpatialIndexItem spatialIndexItem : spatialIndexItems) { - StorageItem storageItem = new StorageItem(spatialIndexItem.getEnvelope(), spatialIndexItem.getItem()); + StorageItem storageItem = new StorageItem(spatialIndexItem.getEnvelope(), spatialIndexItem.getItem().getURI()); storageItems.add(storageItem); } } @@ -62,8 +65,8 @@ public Collection getIndexItems() { return indexItems; } - public SpatialIndex getSpatialIndex() throws SpatialIndexException { - return new SpatialIndex(getIndexItems(), srsURI); + public SpatialIndexV1 getSpatialIndex() throws SpatialIndexException { + return new SpatialIndexV1(getIndexItems(), srsURI); } private class StorageItem implements Serializable { @@ -76,6 +79,11 @@ public StorageItem(Envelope envelope, Resource item) { this.uri = item.getURI(); } + public StorageItem(Envelope envelope, String uri) { + this.envelope = envelope; + this.uri = uri; + } + public Envelope getEnvelope() { return envelope; } diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/compat/SpatialIndexIo.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/compat/SpatialIndexIo.java new file mode 100644 index 00000000000..8bd7f0b70f5 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/compat/SpatialIndexIo.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.compat; + +import java.lang.invoke.MethodHandles; +import java.nio.file.Path; + +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v1.SpatialIndexAdapterV1; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexIoKryo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("removal") +public class SpatialIndexIo { + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** Attempt to load a spatial index from file using all supported formats. */ + public static final SpatialIndex load(Path spatialIndexFile) throws SpatialIndexException { + return load(spatialIndexFile, false); + } + + /** + * Attempt to load a spatial index from file using all supported formats. + * This method should only be used for testing as it allows suppressing warnings when loading legacy index formats. + */ + public static final SpatialIndex load(Path spatialIndexFile, boolean suppressLegacyWarnings) throws SpatialIndexException { + SpatialIndex result; + try { + result = SpatialIndexIoKryo.load(spatialIndexFile); + } catch (Throwable t1) { + if (!suppressLegacyWarnings) { + LOGGER.warn("Failed to load spatial index with latest format. Trying legacy formats...", t1); + } + try { + org.apache.jena.geosparql.spatial.index.v1.SpatialIndexV1 v1 = org.apache.jena.geosparql.spatial.index.v1.SpatialIndexV1.load(spatialIndexFile.toFile()); + result = new SpatialIndexAdapterV1(v1); + + if (!suppressLegacyWarnings) { + LOGGER.warn("Successfully loaded spatial index with legacy format v1. Upgrade advised."); + } + } catch (Throwable t2) { + LOGGER.warn("Failed to load spatial index legacy format.", t2); + t1.addSuppressed(new RuntimeException("Failed to load spatial index with any format.", t2)); + throw t1; + } + } + return result; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v1/SpatialIndexAdapterV1.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v1/SpatialIndexAdapterV1.java new file mode 100644 index 00000000000..9f33eeae159 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v1/SpatialIndexAdapterV1.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v1; + +import java.io.File; +import java.nio.file.Path; +import java.util.Collection; +import java.util.HashSet; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.jena.geosparql.implementation.SRSInfo; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.graph.Node; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.locationtech.jts.geom.Envelope; + +/** Adapter class for spatial index v1. */ +@Deprecated(forRemoval=true) +public class SpatialIndexAdapterV1 + implements SpatialIndex +{ + protected org.apache.jena.geosparql.spatial.index.v1.SpatialIndexV1 v1; + + public SpatialIndexAdapterV1(org.apache.jena.geosparql.spatial.index.v1.SpatialIndexV1 v1) { + super(); + this.v1 = v1; + } + + @Override + public SRSInfo getSrsInfo() { + return v1.getSrsInfo(); + } + + @Override + public boolean isEmpty() { + return v1.isEmpty(); + } + + @Override + public long getSize() { + return -1; + } + + protected Collection adapt(Collection resources) { + Collection result = resources.stream() + .map(RDFNode::asNode) + .collect(Collectors.toCollection(HashSet::new)); + return result; + } + + @Override + public Collection query(Envelope searchEnvelope, Node graphName) { + HashSet resources = v1.query(searchEnvelope); + return adapt(resources); + } + + @Override + public Path getLocation() { + Path result = Optional.ofNullable(v1.getLocation()).map(File::toPath).orElse(null); + return result; + } + + @Override + public void setLocation(Path location) { + v1.setLocation(location == null ? null : location.toFile()); + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v1/SpatialIndexV1.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v1/SpatialIndexV1.java new file mode 100644 index 00000000000..5e5b27b6552 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v1/SpatialIndexV1.java @@ -0,0 +1,578 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v1; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; + +import org.apache.jena.atlas.RuntimeIOException; +import org.apache.jena.atlas.io.IOX; +import org.apache.jena.geosparql.configuration.GeoSPARQLOperations; +import org.apache.jena.geosparql.implementation.GeometryWrapper; +import org.apache.jena.geosparql.implementation.SRSInfo; +import org.apache.jena.geosparql.implementation.registry.SRSRegistry; +import org.apache.jena.geosparql.implementation.vocabulary.Geo; +import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; +import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; +import org.apache.jena.geosparql.spatial.ConvertLatLon; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexConstants; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.SpatialIndexItem; +import org.apache.jena.geosparql.spatial.SpatialIndexStorage; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.DatasetFactory; +import org.apache.jena.query.ReadWrite; +import org.apache.jena.rdf.model.Literal; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.NodeIterator; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.ResIterator; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.StmtIterator; +import org.apache.jena.sparql.engine.ExecutionContext; +import org.apache.jena.sparql.util.Context; +import org.apache.jena.sparql.util.ModelUtils; +import org.apache.jena.util.iterator.ExtendedIterator; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.index.strtree.STRtree; +import org.opengis.geometry.MismatchedDimensionException; +import org.opengis.referencing.operation.TransformException; +import org.opengis.util.FactoryException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * SpatialIndex for testing bounding box collisions between geometries within a + * Dataset.
+ * Queries must be performed using the same SRS URI as the SpatialIndex.
+ * The SpatialIndex is added to the Dataset Context when it is built.
+ * QueryRewriteIndex is also stored in the SpatialIndex as its content is + * Dataset specific. + */ +@Deprecated(forRemoval=true) /** Superseded by {@link SpatialIndex} and {@link org.apache.jena.geosparql.spatial.index.v2.SpatialIndexPerGraph}. */ +public class SpatialIndexV1 { + + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private transient final SRSInfo srsInfo; + private boolean isBuilt; + private final STRtree strTree; + private static final int MINIMUM_CAPACITY = 2; + private File location; + + private SpatialIndexV1() { + this.strTree = new STRtree(MINIMUM_CAPACITY); + this.isBuilt = true; + this.strTree.build(); + this.srsInfo = SRSRegistry.getSRSInfo(SRS_URI.DEFAULT_WKT_CRS84); + } + + /** + * Unbuilt Spatial Index with provided capacity. + * + * @param capacity + * @param srsURI + */ + public SpatialIndexV1(int capacity, String srsURI) { + int indexCapacity = capacity < MINIMUM_CAPACITY ? MINIMUM_CAPACITY : capacity; + this.strTree = new STRtree(indexCapacity); + this.isBuilt = false; + this.srsInfo = SRSRegistry.getSRSInfo(srsURI); + } + + /** + * Built Spatial Index with provided capacity. + * + * @param spatialIndexItems + * @param srsURI + * @throws SpatialIndexException + */ + public SpatialIndexV1(Collection spatialIndexItems, String srsURI) throws SpatialIndexException { + int indexCapacity = spatialIndexItems.size() < MINIMUM_CAPACITY ? MINIMUM_CAPACITY : spatialIndexItems.size(); + this.strTree = new STRtree(indexCapacity); + insertItems(spatialIndexItems); + this.strTree.build(); + this.isBuilt = true; + this.srsInfo = SRSRegistry.getSRSInfo(srsURI); + } + + public File getLocation() { + return location; + } + + public void setLocation(File location) { + this.location = location; + } + + /** + * + * @return Information about the SRS used by the SpatialIndex. + */ + public SRSInfo getSrsInfo() { + return srsInfo; + } + + /** + * + * @return True if the SpatialIndex is empty. + */ + public boolean isEmpty() { + return strTree.isEmpty(); + } + + /** + * + * @return True if the SpatialIndex has been built. + */ + public boolean isBuilt() { + return isBuilt; + } + + /** + * Build the Spatial Index. No more items can be added. + */ + public void build() { + if (!isBuilt) { + strTree.build(); + isBuilt = true; + } + } + + /** + * Items to add to an unbuilt Spatial Index. + * + * @param indexItems + * @throws SpatialIndexException + */ + public final void insertItems(Collection indexItems) throws SpatialIndexException { + + for (SpatialIndexItem indexItem : indexItems) { + insertItem(indexItem.getEnvelope(), ModelUtils.convertGraphNodeToRDFNode(indexItem.getItem()).asResource()); + } + } + + /** + * Item to add to an unbuilt Spatial Index. + * + * @param envelope + * @param item + * @throws SpatialIndexException + */ + public final void insertItem(Envelope envelope, Resource item) throws SpatialIndexException { + if (!isBuilt) { + strTree.insert(envelope, item); + } else { + throw new SpatialIndexException("SpatialIndex has been built and cannot have additional items."); + } + } + + @SuppressWarnings("unchecked") + public HashSet query(Envelope searchEnvelope) { + if (!strTree.isEmpty()) { + return new HashSet<>(strTree.query(searchEnvelope)); + } else { + return new HashSet<>(); + } + } + + @Override + public String toString() { + return "SpatialIndex{" + "srsInfo=" + srsInfo + ", isBuilt=" + isBuilt + ", strTree=" + strTree + '}'; + } + + /** + * Retrieve the SpatialIndex from the Context. + * + * @param execCxt + * @return SpatialIndex contained in the Context. + * @throws SpatialIndexException + */ + public static final SpatialIndexV1 retrieve(ExecutionContext execCxt) throws SpatialIndexException { + + Context context = execCxt.getContext(); + SpatialIndexV1 spatialIndex = (SpatialIndexV1) context.get(SpatialIndexConstants.symSpatialIndex, null); + + if (spatialIndex == null) { + throw new SpatialIndexException("Dataset Context does not contain SpatialIndex."); + } + + return spatialIndex; + } + + /** + * + * @param execCxt + * @return True if a SpatialIndex is defined in the ExecutionContext. + */ + public static final boolean isDefined(ExecutionContext execCxt) { + Context context = execCxt.getContext(); + return context.isDefined(SpatialIndexConstants.symSpatialIndex); + } + + /** + * Set the SpatialIndex into the Context of the Dataset for later retrieval + * and use in spatial functions. + * + * @param dataset + * @param spatialIndex + */ + public static final void setSpatialIndex(Dataset dataset, SpatialIndexV1 spatialIndex) { + Context context = dataset.getContext(); + SpatialIndex wrapper = new SpatialIndexAdapterV1(spatialIndex); + SpatialIndexLib.setSpatialIndex(context, wrapper); + } + + /** + * Build Spatial Index from all graphs in Dataset.
+ * Dataset contains SpatialIndex in Context.
+ * Spatial Index written to file. + * + * @param dataset + * @param srsURI + * @param spatialIndexFile + * @return SpatialIndex constructed. + * @throws SpatialIndexException + */ + public static SpatialIndexV1 buildSpatialIndex(Dataset dataset, String srsURI, File spatialIndexFile) throws SpatialIndexException { + + SpatialIndexV1 spatialIndex = load(spatialIndexFile); + spatialIndex.setLocation(spatialIndexFile); + + if (spatialIndex.isEmpty()) { + Collection spatialIndexItems = findSpatialIndexItems(dataset, srsURI); + save(spatialIndexFile, spatialIndexItems, srsURI); + spatialIndex = new SpatialIndexV1(spatialIndexItems, srsURI); + spatialIndex.build(); + } + + setSpatialIndex(dataset, spatialIndex); + return spatialIndex; + } + + /** + * Build Spatial Index from all graphs in Dataset.
+ * Dataset contains SpatialIndex in Context.
+ * SRS URI based on most frequent found in Dataset.
+ * Spatial Index written to file. + * + * @param dataset + * @param spatialIndexFile + * @return SpatialIndex constructed. + * @throws SpatialIndexException + */ + public static SpatialIndexV1 buildSpatialIndex(Dataset dataset, File spatialIndexFile) throws SpatialIndexException { + String srsURI = GeoSPARQLOperations.findModeSRS(dataset); + SpatialIndexV1 spatialIndex = buildSpatialIndex(dataset, srsURI, spatialIndexFile); + return spatialIndex; + } + + /** + * Build Spatial Index from all graphs in Dataset.
+ * Dataset contains SpatialIndex in Context. + * + * @param dataset + * @param srsURI + * @return SpatialIndex constructed. + * @throws SpatialIndexException + */ + public static SpatialIndexV1 buildSpatialIndex(Dataset dataset, String srsURI) throws SpatialIndexException { + LOGGER.info("Building Spatial Index - Started"); + + Collection items = findSpatialIndexItems(dataset, srsURI); + SpatialIndexV1 spatialIndex = new SpatialIndexV1(items, srsURI); + spatialIndex.build(); + setSpatialIndex(dataset, spatialIndex); + LOGGER.info("Building Spatial Index - Completed"); + return spatialIndex; + } + + /** + * Find Spatial Index Items from all graphs in Dataset.
+ * + * @param dataset + * @param srsURI + * @return SpatialIndexItems found. + * @throws SpatialIndexException + */ + public static Collection findSpatialIndexItems(Dataset dataset, String srsURI) throws SpatialIndexException { + //Default Model + dataset.begin(ReadWrite.READ); + Model defaultModel = dataset.getDefaultModel(); + Collection items = getSpatialIndexItems(defaultModel, srsURI); + + //Named Models + Iterator graphNames = dataset.listNames(); + while (graphNames.hasNext()) { + String graphName = graphNames.next(); + Model namedModel = dataset.getNamedModel(graphName); + Collection graphItems = getSpatialIndexItems(namedModel, srsURI); + items.addAll(graphItems); + } + + dataset.end(); + + return items; + } + + /** + * Build Spatial Index from all graphs in Dataset.
+ * Dataset contains SpatialIndex in Context.
+ * SRS URI based on most frequent found in Dataset. + * + * @param dataset + * @return SpatialIndex constructed. + * @throws SpatialIndexException + */ + public static SpatialIndexV1 buildSpatialIndex(Dataset dataset) throws SpatialIndexException { + String srsURI = GeoSPARQLOperations.findModeSRS(dataset); + SpatialIndexV1 spatialIndex = buildSpatialIndex(dataset, srsURI); + return spatialIndex; + } + + /** + * Wrap Model in a Dataset and build SpatialIndex. + * + * @param model + * @param srsURI + * @return Dataset with default Model and SpatialIndex in Context. + * @throws SpatialIndexException + */ + public static final Dataset wrapModel(Model model, String srsURI) throws SpatialIndexException { + + Dataset dataset = DatasetFactory.createTxnMem(); + dataset.setDefaultModel(model); + buildSpatialIndex(dataset, srsURI); + + return dataset; + } + + /** + * Wrap Model in a Dataset and build SpatialIndex. + * + * @param model + * @return Dataset with default Model and SpatialIndex in Context. + * @throws SpatialIndexException + */ + public static final Dataset wrapModel(Model model) throws SpatialIndexException { + Dataset dataset = DatasetFactory.createTxnMem(); + dataset.setDefaultModel(model); + String srsURI = GeoSPARQLOperations.findModeSRS(dataset); + buildSpatialIndex(dataset, srsURI); + + return dataset; + } + + /** + * Find items from the Model transformed to the SRS URI. + * + * @param model + * @param srsURI + * @return Items found in the Model in the SRS URI. + * @throws SpatialIndexException + */ + public static final Collection getSpatialIndexItems(Model model, String srsURI) throws SpatialIndexException { + + List items = new ArrayList<>(); + + //Only add one set of statements as a converted dataset will duplicate the same info. + if (model.contains(null, Geo.HAS_GEOMETRY_PROP, (Resource) null)) { + LOGGER.info("Feature-hasGeometry-Geometry statements found."); + if (model.contains(null, SpatialExtension.GEO_LAT_PROP, (Literal) null)) { + LOGGER.warn("Lat/Lon Geo predicates also found but will not be added to index."); + } + Collection geometryLiteralItems = getGeometryLiteralIndexItems(model, srsURI); + items.addAll(geometryLiteralItems); + } else if (model.contains(null, SpatialExtension.GEO_LAT_PROP, (Literal) null)) { + LOGGER.info("Geo predicate statements found."); + Collection geoPredicateItems = getGeoPredicateIndexItems(model, srsURI); + items.addAll(geoPredicateItems); + } + + return items; + } + + /** + * + * @param model + * @param srsURI + * @return GeometryLiteral items prepared for adding to SpatialIndex. + * @throws SpatialIndexException + */ + private static Collection getGeometryLiteralIndexItems(Model model, String srsURI) throws SpatialIndexException { + List items = new ArrayList<>(); + StmtIterator stmtIt = model.listStatements(null, Geo.HAS_GEOMETRY_PROP, (Resource) null); + while (stmtIt.hasNext()) { + Statement stmt = stmtIt.nextStatement(); + + Resource feature = stmt.getSubject(); + Resource geometry = stmt.getResource(); + + ExtendedIterator nodeIter = model.listObjectsOfProperty(geometry, Geo.HAS_SERIALIZATION_PROP); + if (!nodeIter.hasNext()) { + NodeIterator wktNodeIter = model.listObjectsOfProperty(geometry, Geo.AS_WKT_PROP); + NodeIterator gmlNodeIter = model.listObjectsOfProperty(geometry, Geo.AS_GML_PROP); + nodeIter = wktNodeIter.andThen(gmlNodeIter); + } + + while (nodeIter.hasNext()) { + Literal geometryLiteral = nodeIter.next().asLiteral(); + GeometryWrapper geometryWrapper = GeometryWrapper.extract(geometryLiteral); + + try { + //Ensure all entries in the target SRS URI. + GeometryWrapper transformedGeometryWrapper = geometryWrapper.convertSRS(srsURI); + + Envelope envelope = transformedGeometryWrapper.getEnvelope(); + SpatialIndexItem item = new SpatialIndexItem(envelope, feature); + items.add(item); + } catch (FactoryException | MismatchedDimensionException | TransformException ex) { + throw new SpatialIndexException("Transformation Exception: " + geometryLiteral + ". " + ex.getMessage()); + } + + } + } + return items; + } + + /** + * + * @param model + * @param srsURI + * @return Geo predicate objects prepared for adding to SpatialIndex. + */ + private static Collection getGeoPredicateIndexItems(Model model, String srsURI) throws SpatialIndexException { + List items = new ArrayList<>(); + ResIterator resIt = model.listResourcesWithProperty(SpatialExtension.GEO_LAT_PROP); + + while (resIt.hasNext()) { + Resource feature = resIt.nextResource(); + + Literal lat = feature.getRequiredProperty(SpatialExtension.GEO_LAT_PROP).getLiteral(); + Literal lon = feature.getProperty(SpatialExtension.GEO_LON_PROP).getLiteral(); + if (lon == null) { + LOGGER.warn("Geo predicates: latitude found but not longitude. " + feature); + continue; + } + + Literal latLonPoint = ConvertLatLon.toLiteral(lat.getFloat(), lon.getFloat()); + GeometryWrapper geometryWrapper = GeometryWrapper.extract(latLonPoint); + + try { + //Ensure all entries in the target SRS URI. + GeometryWrapper transformedGeometryWrapper = geometryWrapper.convertSRS(srsURI); + + Envelope envelope = transformedGeometryWrapper.getEnvelope(); + SpatialIndexItem item = new SpatialIndexItem(envelope, feature); + items.add(item); + } catch (FactoryException | MismatchedDimensionException | TransformException ex) { + throw new SpatialIndexException("Transformation Exception: " + geometryWrapper.getLexicalForm() + ". " + ex.getMessage()); + } + } + return items; + } + + /** + * Load a SpatialIndex from file.
+ * Index will be built and empty if file does not exist or is null. + * + * @param spatialIndexFile + * @return Built Spatial Index. + * @throws SpatialIndexException + */ + public static final SpatialIndexV1 load(File spatialIndexFile) throws SpatialIndexException { + + if (spatialIndexFile != null && spatialIndexFile.exists()) { + LOGGER.info("Loading Spatial Index - Started: {}", spatialIndexFile.getAbsolutePath()); + //Cannot directly store the SpatialIndex due to Resources not being serializable, use SpatialIndexStorage class. + try (ObjectInputStream in = new ObjectInputStream(new FileInputStream(spatialIndexFile))) { + SpatialIndexStorage storage = (SpatialIndexStorage) in.readObject(); + + SpatialIndexV1 spatialIndex = storage.getSpatialIndex(); + LOGGER.info("Loading Spatial Index - Completed: {}", spatialIndexFile.getAbsolutePath()); + return spatialIndex; + } catch (ClassNotFoundException | IOException ex) { + throw new SpatialIndexException("Loading Exception: " + ex.getMessage(), ex); + } + } else { + return new SpatialIndexV1(); + } + } + + /** + * Save SpatialIndex contents to file. + * + * @param spatialIndexFileURI + * @param spatialIndexItems + * @param srsURI + * @throws SpatialIndexException + */ + public static final void save(String spatialIndexFileURI, Collection spatialIndexItems, String srsURI) throws SpatialIndexException { + save(new File(spatialIndexFileURI), spatialIndexItems, srsURI); + } + + /** + * Save SpatialIndex contents to file. + * + * @param spatialIndexFile + * @param spatialIndexItems + * @param srsURI + * @throws SpatialIndexException + */ + public static final void save(File spatialIndexFile, Collection spatialIndexItems, String srsURI) throws SpatialIndexException { + + //Cannot directly store the SpatialIndex due to Resources not being serializable, use SpatialIndexStorage class. + if (spatialIndexFile != null) { + LOGGER.info("Saving Spatial Index - Started: {}", spatialIndexFile.getAbsolutePath()); + SpatialIndexStorage storage = new SpatialIndexStorage(spatialIndexItems, srsURI); + String filename = spatialIndexFile.getAbsolutePath(); + Path file = Path.of(filename); + Path tmpFile = IOX.uniqueDerivedPath(file, null); + try { + Files.deleteIfExists(file); + } catch (IOException ex) { + throw new SpatialIndexException("Failed to delete file: " + ex.getMessage()); + } + try { + IOX.safeWriteOrCopy(file, tmpFile, + out->{ + ObjectOutputStream oos = new ObjectOutputStream(out); + oos.writeObject(storage); + oos.flush(); + }); + } catch (RuntimeIOException ex) { + throw new SpatialIndexException("Save Exception: " + ex.getMessage()); + } finally { + LOGGER.info("Saving Spatial Index - Completed: {}", spatialIndexFile.getAbsolutePath()); + } + } + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/GeometryGenerator.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/GeometryGenerator.java new file mode 100644 index 00000000000..f07e8018156 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/GeometryGenerator.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; + +import org.apache.jena.geosparql.implementation.GeometryWrapper; +import org.apache.jena.geosparql.implementation.datatype.WKTDatatype; +import org.apache.jena.geosparql.implementation.jts.CustomGeometryFactory; +import org.apache.jena.geosparql.implementation.vocabulary.Geo; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Polygon; + +/** + * Class to generate various geometry types for testing. The generated coordinate sequences are based on circles. + * This class is also used in jena-benchmarks. + */ +public class GeometryGenerator { + + public enum GeometryType { + POINT, + LINESTRING, + LINEARRING, + POLYGON, + MULTIPOINT, + MULTILINESTRING, + MULTIPOLYGON, + GEOMETRYCOLLECTION; + } + + public static Coordinate[] createCircle(GeometryFactory geometryFactory, Coordinate center, double radius, int numPoints, boolean closed) { + Coordinate[] coords = new Coordinate[numPoints + (closed ? 1 : 0)]; // +1 to close the circle + for (int i = 0; i < numPoints; i++) { + double angle = 2 * Math.PI * i / numPoints; + double x = center.x + radius * Math.cos(angle); + double y = center.y + radius * Math.sin(angle); + coords[i] = new Coordinate(x, y); + } + if (closed) { + coords[numPoints] = coords[0]; + } + return coords; + } + + public static Geometry createGeom(GeometryType type, GeometryFactory f, Coordinate center, double radius, int numPoints) { + Geometry result; + switch(type) { + case POINT: + // Only return the point - (a circle of radius 0) + result = f.createPoint(center); break; + default: + // Non-multi geometries. + Coordinate[] c1 = createCircle(f,center, radius, numPoints, true); + + switch(type) { + case MULTIPOINT: result = f.createMultiPointFromCoords(c1); break; + case LINESTRING: result = f.createLineString(c1); break; + case LINEARRING: result = f.createLinearRing(c1); break; + case POLYGON: result = f.createPolygon(c1); break; + default: + // Multi geometries. + Coordinate[] c2 = createCircle(f,center, radius * 0.5, (int)Math.max(3, numPoints * 0.8), true); + + switch(type) { + case MULTILINESTRING: result = f.createMultiLineString(new LineString[]{f.createLineString(c1), f.createLineString(c2)}); break; + case MULTIPOLYGON: result = f.createMultiPolygon(new Polygon[]{f.createPolygon(c1), f.createPolygon(c2)}); break; + case GEOMETRYCOLLECTION: result = f.createGeometryCollection(new Geometry[] {f.createPoint(center), f.createLinearRing(c2)}); break; + default: + throw new RuntimeException("Unsupported geometry type: " + type); + } + } + } + return result; + } + + /** + * Create a map that maps each geometry type to the specified amount. + * For use with {@link #generateGraph(Graph, Envelope, Map)}. + */ + public static Map createConfig(long amount) { + // How many geometries to generate of each type. + // Note that the index only stores their envelopes. + Map config = new LinkedHashMap<>(); + config.put(GeometryType.POINT, amount); + config.put(GeometryType.LINESTRING, amount); + config.put(GeometryType.LINEARRING, amount); + config.put(GeometryType.POLYGON, amount); + config.put(GeometryType.MULTIPOINT, amount); + config.put(GeometryType.MULTILINESTRING, amount); + config.put(GeometryType.MULTIPOLYGON, amount); + config.put(GeometryType.GEOMETRYCOLLECTION, amount); + return config; + } + + /** + * Generate GeoSPARQL data with various geometry types in the given graph. + * + * @param graph The target graph. + * @param envelope The allowed area for the positions of generated geometries (the shape may overlap). + * @param config Map of geometry type to the number of geometries to generate. + * + * @return The number of generated geometries added to the graph. + */ + public static long generateGraph(Graph graph, Envelope envelope, Map config) { + GeometryFactory geometryFactory = CustomGeometryFactory.theInstance(); + + float maxRadius = 1; + int minNumPoints = 3; + int maxNumPoints = 30; + + Random rand = new Random(0); + + float minX = (float)envelope.getMinX(); + float maxX = (float)envelope.getMaxX(); + float minY = (float)envelope.getMinY(); + float maxY = (float)envelope.getMaxY(); + + float dX = maxX - minX; + float dY = maxY - minY; + int dNumPoints = maxNumPoints - minNumPoints; + + long generatedItemCount = 0; + long nextFeatureId = 0; + for (Entry e : config.entrySet()) { + GeometryType geometryType = e.getKey(); + long count = e.getValue().longValue(); + for (long i = 0; i < count; ++i) { + + float x = minX + rand.nextFloat(dX); + float y = minY + rand.nextFloat(dY); + int numPoints = minNumPoints + (int)(rand.nextFloat(dNumPoints)); + + Coordinate c = new Coordinate(x, y); + Geometry g = GeometryGenerator.createGeom(geometryType, geometryFactory, c, maxRadius, numPoints); + + long featureId = nextFeatureId++; + Node feature = NodeFactory.createURI("http://www.example.org/feature" + featureId); + Node geometry = NodeFactory.createURI("http://www.example.org/geometry" + featureId); + Node geom = new GeometryWrapper(g, WKTDatatype.URI).asNode(); + + graph.add(feature, Geo.HAS_GEOMETRY_NODE, geometry); + graph.add(geometry, Geo.AS_WKT_NODE, geom); + + ++generatedItemCount; + } + } + return generatedItemCount; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/KryoRegistratorSpatialIndexV2.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/KryoRegistratorSpatialIndexV2.java new file mode 100644 index 00000000000..a11738f4afb --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/KryoRegistratorSpatialIndexV2.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.jena.geosparql.kryo.EnvelopeSerializer; +import org.apache.jena.geosparql.kryo.NodeSerializer; +import org.apache.jena.geosparql.kryo.TripleSerializer; +import org.apache.jena.graph.Triple; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryCollection; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.LinearRing; +import org.locationtech.jts.geom.MultiLineString; +import org.locationtech.jts.geom.MultiPoint; +import org.locationtech.jts.geom.MultiPolygon; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.index.strtree.STRtree; +import org.locationtech.jts.index.strtree.STRtreeSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.serializers.MapSerializer; + +/** + * The class is used to configure the kryo serialization + * of the spatial index. Changes to the kryo configuration may + * break loading of existing index files - so changes must be made + * with care. + */ +public class KryoRegistratorSpatialIndexV2 { + + private final static Logger LOGGER = LoggerFactory.getLogger(KryoRegistratorSpatialIndexV2.class); + + public static void registerClasses(Kryo kryo, Serializer geometrySerializer) { + LOGGER.debug("Registering kryo serializers for spatial index v2."); + + // Java + Serializer mapSerializer = new MapSerializer(); + kryo.register(Map.class, mapSerializer); + kryo.register(HashMap.class, mapSerializer); + kryo.register(LinkedHashMap.class, mapSerializer); + + // Jena + NodeSerializer.register(kryo); + registerTripleSerializer(kryo); // Needed for RDFstar Nodes. + + // JTS + kryo.register(STRtree.class, new STRtreeSerializer()); + kryo.register(Envelope.class, new EnvelopeSerializer()); + + // Jena + JTS + kryo.register(STRtreePerGraph.class, new STRtreePerGraphSerializer()); + + // The index only stores envelopes and jena nodes. + // Therefore, geometry serializers should not be needed. + if (geometrySerializer != null) { + registerGeometrySerializers(kryo, geometrySerializer); + } + } + + /** + * The default serializer for {@link Triple}. + * Must be registered in addition to node serializers for RDF-Star (triples-in-nodes). + */ + public static void registerTripleSerializer(Kryo kryo) { + kryo.register(Triple.class, new TripleSerializer()); + + // Array-of-triples serializer variant would have to be registered separately - but does not seem to be needed. + // kryo.register(Triple[].class); + } + + public static void registerGeometrySerializers(Kryo kryo, Serializer geometrySerializer) { + kryo.register(Point.class, geometrySerializer); + kryo.register(LinearRing.class, geometrySerializer); + kryo.register(LineString.class, geometrySerializer); + kryo.register(Polygon.class, geometrySerializer); + kryo.register(MultiPoint.class, geometrySerializer); + kryo.register(MultiLineString.class, geometrySerializer); + kryo.register(MultiPolygon.class, geometrySerializer); + kryo.register(GeometryCollection.class, geometrySerializer); + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreePerGraph.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreePerGraph.java new file mode 100644 index 00000000000..6d534bb9a04 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreePerGraph.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.jena.graph.Node; +import org.apache.jena.sparql.core.Quad; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.index.strtree.STRtree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class STRtreePerGraph { + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Mapping of graph node to STRtree. + * + * @implNote + * The STRtree that corresponds to the default graph (referred to as 'default tree') + * uses the name {@link Quad#defaultGraphIRI}. + * The default tree is maintained as part of the treeMap (rather than as a separate field) + * to simplify merging of partial spatial indexes. + */ + private Map treeMap; + + /** Unmodifiable view of treeMap */ + private Map treeMapView; + + private boolean isBuilt = false; + + public STRtreePerGraph() { + this(new ConcurrentHashMap<>()); + } + + public STRtreePerGraph(STRtree defaultTree) { + this(new ConcurrentHashMap<>()); + Objects.requireNonNull(defaultTree); + setDefaultTree(defaultTree); + } + + protected STRtreePerGraph(Map treeMap) { + super(); + this.treeMap = Objects.requireNonNull(treeMap); + this.treeMapView = Collections.unmodifiableMap(this.treeMap); + } + + public STRtree getDefaultTree() { + return treeMap.get(Quad.defaultGraphIRI); + } + + /** Returns an unmodifiable view of the tree map. */ + public Map getTreeMap() { + return treeMapView; + } + + /** For serialization. */ + Map getInternalTreeMap() { + return treeMap; + } + + /** Whether a tree with the given name exists. Handles default tree names. */ + public boolean contains(Node name) { + return getTree(name) != null; + } + + /** Whether a tree with the given name exists. Handles default tree names. */ + public STRtree getTree(Node name) { + STRtree result = (name == null || Quad.isDefaultGraph(name)) + ? treeMap.get(Quad.defaultGraphIRI) + : treeMap.get(name); + return result; + } + + /** Whether a tree with the given name exists. Handles default tree names. */ + public STRtree setTree(Node name, STRtree tree) { + STRtree result; + if (name == null || Quad.isDefaultGraph(name)) { + result = setDefaultTree(tree); + } else { + result = setNamedTree(name, tree); + } + return result; + } + + /** Returns the prior default tree. */ + public STRtree setDefaultTree(STRtree tree) { + return setNamedTree(Quad.defaultGraphIRI, tree); + } + + /** Returns the prior tree of graphNode. */ + protected STRtree setNamedTree(Node graphNode, STRtree tree) { + Objects.requireNonNull(graphNode); // Default graph must name must be Quad.defaultGraphIRI. + return treeMap.put(graphNode, tree); + } + + /** Add all data of 'other' to this. Builds the added trees if {@link #isBuilt()} is true. */ + public void setTrees(Map treeMap) { + // If the index is already built then build all trees being added. + if (isBuilt()) { + treeMap.values().forEach(STRtree::build); + } + this.treeMap.putAll(treeMap); + } + + public boolean removeTree(Node node) { + if (node == null) { + node = Quad.defaultGraphIRI; + } + return treeMap.remove(node) != null; + } + + @SuppressWarnings("unchecked") + public Collection queryOneGraph(Envelope searchEnvelope, Node graph) { + Collection result; + if (graph == null || Quad.isDefaultGraph(graph)) { + // Handle default graph. + STRtree defaultTree = getDefaultTree(); + result = defaultTree == null + ? Set.of() + : new LinkedHashSet<>(defaultTree.query(searchEnvelope)); + } else if (Quad.isUnionGraph(graph)) { + // Handle union graph (avoid). + LOGGER.warn("spatial index lookup on union graph"); + result = treeMap.entrySet().stream() + .filter(e -> !Quad.isDefaultGraph(e.getKey())) // Exclude default graph. + .map(Entry::getValue) + .map(tree -> tree.query(searchEnvelope)) + .collect(LinkedHashSet::new, + Set::addAll, + Set::addAll); + } else { + // Handle specific named graph. + STRtree tree = treeMap.get(graph); + if (tree == null) { + LOGGER.warn("graph not indexed: " + graph); + } + if (tree != null && !tree.isEmpty()) { + result = new LinkedHashSet<>(tree.query(searchEnvelope)); + } else { + result = new HashSet<>(); + } + } + return result; + } + + public boolean isEmpty() { + boolean result = treeMap.values().stream().allMatch(STRtree::isEmpty); + return result; + } + + public void build() { + if (!isBuilt) { + treeMap.values().forEach(STRtree::build); + isBuilt = true; + } + } + + public boolean isBuilt() { + return isBuilt; + } + + public long size() { + long result = treeMap.values().stream().mapToLong(STRtree::size).sum(); + return result; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreePerGraphSerializer.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreePerGraphSerializer.java new file mode 100644 index 00000000000..38ba56c7ee7 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreePerGraphSerializer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.util.Map; + +import org.apache.jena.graph.Node; +import org.locationtech.jts.index.strtree.STRtree; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +public class STRtreePerGraphSerializer + extends Serializer +{ + @Override + public void write(Kryo kryo, Output output, STRtreePerGraph index) { + output.writeBoolean(index.isBuilt()); + kryo.writeClassAndObject(output, index.getInternalTreeMap()); + } + + @Override + public STRtreePerGraph read(Kryo kryo, Input input, Class type) { + boolean isBuilt = input.readBoolean(); + @SuppressWarnings("unchecked") + Map treeMap = (Map)kryo.readClassAndObject(input); + STRtreePerGraph result = new STRtreePerGraph(treeMap); + if (isBuilt) { + result.build(); + } + return result; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreeUtils.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreeUtils.java new file mode 100644 index 00000000000..37775eae533 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/STRtreeUtils.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.jena.atlas.iterator.Iter; +import org.apache.jena.atlas.iterator.IteratorCloseable; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.SpatialIndexFindUtils; +import org.apache.jena.geosparql.spatial.SpatialIndexItem; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.Quad; +import org.locationtech.jts.index.strtree.STRtree; + +public class STRtreeUtils { + static final int MINIMUM_CAPACITY = 2; + + public static STRtree buildSpatialIndexTree(Graph graph, String srsURI) throws SpatialIndexException { + try { + STRtree tree; + IteratorCloseable it = SpatialIndexFindUtils.findSpatialIndexItems(graph, srsURI); + try { + tree = buildSpatialIndexTree(it); + } finally { + it.close(); + } + return tree; + } catch (Throwable e) { + throw new SpatialIndexException("Spatial index construction failed.", e); + } + } + + // XXX This method overlaps function-wise with SpatialIndexerComputation. Consolidate? + public static STRtreePerGraph buildSpatialIndexTree(DatasetGraph datasetGraph, String srsURI) throws SpatialIndexException { + Map treeMap = new LinkedHashMap<>(); + + // Process default graph. + // LOGGER.info("building spatial index for default graph ..."); + Graph defaultGraph = datasetGraph.getDefaultGraph(); + STRtree defaultGraphTree = buildSpatialIndexTree(defaultGraph, srsURI); + treeMap.put(Quad.defaultGraphIRI, defaultGraphTree); + + // Process named graphs. + Iterator graphIter = datasetGraph.listGraphNodes(); + try { + while (graphIter.hasNext()) { + Node graphNode = graphIter.next(); + // LOGGER.info("building spatial index for graph {} ...", graphNode); + Graph namedGraph = datasetGraph.getGraph(graphNode); + treeMap.put(graphNode, buildSpatialIndexTree(namedGraph, srsURI)); + } + } finally { + Iter.close(graphIter); + } + + return new STRtreePerGraph(treeMap); + } + + /** + * Create an STRtree from the elements of the given iterator. + * It's the caller's responsibility to close the iterator if needed. + */ + public static STRtree buildSpatialIndexTree(Iterator it) throws SpatialIndexException { + // Collecting items into a list in order to assist tree construction with a known size. + List items = Iter.toList(it); + STRtree tree = buildSpatialIndexTree(items); + return tree; + } + + public static STRtree buildSpatialIndexTree(Collection items) throws SpatialIndexException { + STRtree tree = new STRtree(Math.max(MINIMUM_CAPACITY, items.size())); + addToTree(tree, items.iterator()); + tree.build(); + return tree; + } + + /** + * Accumulate the elements of the given iterator into an existing (unbuilt) STRtree. + * It's the caller's responsibility to close the iterator if needed. + */ + public static void addToTree(STRtree treeAcc, Iterator it) throws SpatialIndexException { + it.forEachRemaining(item -> treeAcc.insert(item.getEnvelope(), item.getItem())); + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexHeader.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexHeader.java new file mode 100644 index 00000000000..3b3b1b1f825 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexHeader.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.util.Objects; + +import com.google.gson.JsonObject; + +/** + * The header for a spatial index. + * For extensibility, this is merely a view over a JSON object. + */ +public class SpatialIndexHeader { + public static final String TYPE_KEY = "type"; + public static final String VERSION_KEY = "version"; + public static final String SRS_KEY = "srs"; + public static final String GEOMETRY_SERIALIZER_KEY = "geometrySerializer"; + + public static final String TYPE_VALUE = "jena-spatial-index"; + + protected JsonObject json; + + public SpatialIndexHeader() { + this(new JsonObject()); + } + + public SpatialIndexHeader(JsonObject json) { + super(); + this.json = Objects.requireNonNull(json); + } + + public JsonObject getJson() { + return json; + } + + public void setType(String type) { + getJson().addProperty(TYPE_KEY, type); + } + + public String getType() { + return getJson().get(TYPE_KEY).getAsString(); + } + + public void setVersion(String version) { + getJson().addProperty(VERSION_KEY, version); + } + + public String getVersion() { + return getJson().get(VERSION_KEY).getAsString(); + } + + public void setSrsUri(String type) { + getJson().addProperty(SRS_KEY, type); + } + + public String getSrsUri() { + return getJson().get(SRS_KEY).getAsString(); + } + + public void setGeometrySerializerClass(String className) { + getJson().addProperty(GEOMETRY_SERIALIZER_KEY, className); + } + + public String getGeometrySerializerClass() { + return getJson().get(GEOMETRY_SERIALIZER_KEY).getAsString(); + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexIoKryo.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexIoKryo.java new file mode 100644 index 00000000000..834c1cc8a6d --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexIoKryo.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +import org.apache.jena.atlas.RuntimeIOException; +import org.apache.jena.atlas.io.IOX; +import org.apache.jena.geosparql.implementation.SRSInfo; +import org.apache.jena.geosparql.implementation.registry.SRSRegistry; +import org.apache.jena.geosparql.kryo.GeometrySerializerJtsWkb; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.query.Dataset; +import org.locationtech.jts.geom.Geometry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.google.gson.Gson; +import com.google.gson.JsonObject; + +public class SpatialIndexIoKryo { + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + // Flag whether additional serializers for storing geometries in the index should be registered. + // The current version of the index only stores the envelopes so this feature is not needed. + private static boolean enableGeometrySerde = false; + + public static SpatialIndex loadOrBuildSpatialIndex(Dataset dataset, Path spatialIndexFile) throws SpatialIndexException { + SpatialIndex spatialIndex = loadOrBuildSpatialIndex(dataset, null, spatialIndexFile); + return spatialIndex; + } + + private static boolean isNonEmptyFile(Path path) { + boolean result = false; + if (path != null && Files.exists(path)) { + try { + result = Files.size(path) > 0; + } catch (IOException e) { + throw IOX.exception(e); + } + } + return result; + } + + public static SpatialIndex loadOrBuildSpatialIndex(Dataset dataset, + String srsURI, + Path spatialIndexFile) throws SpatialIndexException { + SpatialIndexPerGraph spatialIndex = null; + + // If the spatial index file exists and has non-zero size then load it. + // Otherwise build one. + if (isNonEmptyFile(spatialIndexFile)) { + spatialIndex = load(spatialIndexFile); + SpatialIndexLib.setSpatialIndex(dataset, spatialIndex); + } else { + spatialIndex = buildSpatialIndex(dataset, srsURI, spatialIndexFile); + } + + return spatialIndex; + } + + public static SpatialIndexPerGraph buildSpatialIndex(Dataset dataset, + String srsURI, + Path spatialIndexFile) throws SpatialIndexException { + SpatialIndexPerGraph spatialIndex = SpatialIndexLib.buildSpatialIndex(dataset.asDatasetGraph(), srsURI); + if (spatialIndexFile != null) { + // Register the source file with the index. + spatialIndex.setLocation(spatialIndexFile); + save(spatialIndexFile, spatialIndex); + } + return spatialIndex; + } + + /** + * Save SpatialIndex to file. + * + * @param spatialIndexFile the file being saved to + * @param index the spatial index + * @throws SpatialIndexException + */ + public static final void save(Path spatialIndexFile, SpatialIndexPerGraph index) throws SpatialIndexException { + Path originalFile = spatialIndexFile.toAbsolutePath(); + LOGGER.info("Saving Spatial Index - Started: " + originalFile); + + // Create a temporary file for writing the new index. + Path tmpFile = IOX.uniqueDerivedPath(originalFile, null); + + // As long as the new file has not been successfully written: + // Move the original file out of the way but don't delete it yet. + Path originalBackup = IOX.uniqueDerivedPath(originalFile, baseName -> baseName + ".bak"); + if (Files.exists(originalFile)) { + IOX.moveAllowCopy(originalFile, originalBackup); + } + + try { + IOX.safeWriteOrCopy(originalFile, tmpFile, out -> writeToOutputStream(out, index)); + LOGGER.info("Saving Spatial Index - Success: " + originalFile); + } catch (RuntimeIOException ex) { + LOGGER.info("Failure writing spatial index: " + originalFile, ex); + // Attempt to restore original file from backed up one. + try { + IOX.moveAllowCopy(originalBackup, originalFile); + } catch (RuntimeException ex2) { + LOGGER.warn("Failed to restore " + originalFile + " + from backup file " + originalBackup, ex2); + } + + throw new SpatialIndexException("Save Exception: " + originalFile + " (via temp file: " + tmpFile + ")", ex); + } + + // Delete backup + try { + Files.deleteIfExists(originalBackup); + } catch (IOException ex) { + LOGGER.warn("Failed to remove no longer needed backup: " + originalBackup, ex); + } + } + + /** + * Write spatial index as Kryo serialization to given OutputStream. + * @param os output stream + * @param index spatial index + */ + public static void writeToOutputStream(OutputStream os, SpatialIndexPerGraph index) { + SpatialIndexHeader header = new SpatialIndexHeader(); + header.setType(SpatialIndexHeader.TYPE_VALUE); + header.setVersion("2.0.0"); + header.setSrsUri(index.getSrsInfo().getSrsURI()); + + GeometrySerializerJtsWkb geometrySerializer = null; + if (enableGeometrySerde) { + geometrySerializer = new GeometrySerializerJtsWkb(); + header.setGeometrySerializerClass(geometrySerializer.getClass().getName()); + } + + Kryo kryo = new Kryo(); + KryoRegistratorSpatialIndexV2.registerClasses(kryo, geometrySerializer); + try (Output output = new Output(os)) { + writeHeader(output, header); + STRtreePerGraph trees = index.getIndex(); + kryo.writeObject(output, trees); + output.flush(); + } + } + + public static void writeHeader(Output output, SpatialIndexHeader header) { + Gson gson = new Gson(); + String headerStr = gson.toJson(header.getJson()); + output.writeString(headerStr); + } + + /** Read a string from input and return it as JSON. */ + public static SpatialIndexHeader readHeader(Input input) { + String headerStr = input.readString(); + Gson gson = new Gson(); + JsonObject obj = gson.fromJson(headerStr, JsonObject.class); + return new SpatialIndexHeader(obj); + } + + /** + * Load a SpatialIndex from file.
+ * Index will be built and empty if file does not exist or is null. + * + * @param spatialIndexFile + * @return Built Spatial Index. + * @throws SpatialIndexException + */ + @SuppressWarnings("unchecked") + public static final SpatialIndexPerGraph load(Path spatialIndexFile) throws SpatialIndexException { + String srsUri; + STRtreePerGraph index; + + spatialIndexFile = spatialIndexFile.toAbsolutePath(); + LOGGER.info("Loading Spatial Index - Started: {}", spatialIndexFile); + + try (Input input = new Input(Files.newInputStream(spatialIndexFile))) { + SpatialIndexHeader header = readHeader(input); + + String type = header.getType(); + if (!"jena-spatial-index".equals(type)) { + throw new RuntimeException("Type does not indicate a spatial index file."); + } + + String version = header.getVersion(); + if (!"2.0.0".equals(version)) { + throw new SpatialIndexException("The version of the spatial index does not match the version of this loader class."); + } + + srsUri = header.getSrsUri(); + + Serializer geometrySerializer = null; + if (enableGeometrySerde) { + String geometrySerdeName = header.getGeometrySerializerClass(); + Objects.requireNonNull(geometrySerdeName, "Field 'geometrySerde' not set."); + + try { + Class geometrySerdeClass = Class.forName(geometrySerdeName); + geometrySerializer = (Serializer)geometrySerdeClass.getConstructor().newInstance(); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException | ClassCastException | SecurityException e) { + throw new SpatialIndexException("Failed to load index", e); + } + } + + Kryo kryo = new Kryo(); + KryoRegistratorSpatialIndexV2.registerClasses(kryo, geometrySerializer); + + index = kryo.readObject(input, STRtreePerGraph.class); + LOGGER.info("Loading Spatial Index - Completed: {}", spatialIndexFile); + } catch (IOException ex) { + throw new SpatialIndexException("Loading Exception: " + ex.getMessage(), ex); + } + + SRSInfo srsInfo = SRSRegistry.getSRSInfo(srsUri); + SpatialIndexPerGraph spatialIndex = new SpatialIndexPerGraph(srsInfo, index, spatialIndexFile); + spatialIndex.setLocation(spatialIndexFile); + return spatialIndex; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexLib.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexLib.java new file mode 100644 index 00000000000..bcb35c47409 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexLib.java @@ -0,0 +1,409 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.geosparql.spatial.index.v2; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import org.apache.jena.atlas.iterator.Iter; +import org.apache.jena.geosparql.configuration.GeoSPARQLOperations; +import org.apache.jena.geosparql.implementation.SRSInfo; +import org.apache.jena.geosparql.implementation.registry.SRSRegistry; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexConstants; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.task.TaskThread; +import org.apache.jena.geosparql.spatial.task.BasicTask; +import org.apache.jena.geosparql.spatial.task.BasicTask.TaskListener; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.DatasetFactory; +import org.apache.jena.query.TxnType; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.NamedGraph; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.sparql.engine.ExecutionContext; +import org.apache.jena.sparql.util.Context; +import org.apache.jena.system.AutoTxn; +import org.apache.jena.system.Txn; +import org.locationtech.jts.index.strtree.STRtree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** ARQ-level utils for Dataset and Context. */ +public class SpatialIndexLib { + + private static final Logger logger = LoggerFactory.getLogger(SpatialIndexLib.class); + + /** Set the preferred SRS for the given (dataset) context. */ + public static final String getPreferredSRS(Context context) { + return context == null ? null : context.getAsString(SpatialIndexConstants.symSrsUri); + } + + /** Set a preferred SRS for the given (dataset) context. This SRS will be used when constructing a spatial index for the first time. */ + public static final void setPreferredSRS(Context context, String srsUri) { + context.set(SpatialIndexConstants.symSrsUri, srsUri); + } + + /** + * Set the SpatialIndex into the Context of the Dataset for later retrieval + * and use in spatial functions. + * + * @param dataset + * @param spatialIndex + */ + public static final void setSpatialIndex(Dataset dataset, SpatialIndex spatialIndex) { + Context cxt = dataset.getContext(); + setSpatialIndex(cxt, spatialIndex); + } + + public static final void setSpatialIndex(DatasetGraph datasetGraph, SpatialIndex spatialIndex) { + Context cxt = datasetGraph.getContext(); + setSpatialIndex(cxt, spatialIndex); + } + + public static final void setSpatialIndex(Context context, SpatialIndex spatialIndex) { + context.set(SpatialIndexConstants.symSpatialIndex, spatialIndex); + } + + public static final T getSpatialIndex(Context cxt) { + return cxt == null ? null : cxt.get(SpatialIndexConstants.symSpatialIndex); + } + + public static final T getSpatialIndex(DatasetGraph dsg) { + return getSpatialIndex(dsg.getContext()); + } + + /** + * + * @param execCxt + * @return True if a SpatialIndex is defined in the ExecutionContext. + */ + public static final boolean isDefined(ExecutionContext execCxt) { + Context context = execCxt.getContext(); + return context.isDefined(SpatialIndexConstants.symSpatialIndex); + } + + /** + * Retrieve the SpatialIndex from the Context. + * + * @param execCxt + * @return SpatialIndex contained in the Context. + * @throws SpatialIndexException + */ + public static final SpatialIndex retrieve(ExecutionContext execCxt) throws SpatialIndexException { + Context context = execCxt.getContext(); + SpatialIndex spatialIndex = (SpatialIndex) context.get(SpatialIndexConstants.symSpatialIndex, null); + if (spatialIndex == null) { + throw new SpatialIndexException("Dataset Context does not contain SpatialIndex."); + } + return spatialIndex; + } + + /** + * Wrap Model in a Dataset and build SpatialIndex. + * + * @param model + * @param srsURI + * @return Dataset with default Model and SpatialIndex in Context. + * @throws SpatialIndexException + */ + public static final Dataset wrapModel(Model model, String srsURI) throws SpatialIndexException { + Dataset dataset = DatasetFactory.createTxnMem(); + dataset.setDefaultModel(model); + buildSpatialIndex(dataset.asDatasetGraph(), srsURI); + return dataset; + } + + /** + * Wrap Model in a Dataset and build SpatialIndex. + * + * @param model + * @return Dataset with default Model and SpatialIndex in Context. + * @throws SpatialIndexException + */ + public static final Dataset wrapModel(Model model) throws SpatialIndexException { + Dataset dataset = DatasetFactory.createTxnMem(); + dataset.setDefaultModel(model); + String srsURI = GeoSPARQLOperations.findModeSRS(dataset); + buildSpatialIndex(dataset.asDatasetGraph(), srsURI); + return dataset; + } + + /** + * Build Spatial Index from all graphs in Dataset.
+ * Dataset contains SpatialIndex in Context.
+ * SRS URI based on most frequent found in Dataset. + * + * @param datasetGraph + * @return SpatialIndex constructed. + * @throws SpatialIndexException + */ + public static SpatialIndex buildSpatialIndex(DatasetGraph datasetGraph) throws SpatialIndexException { + SpatialIndex spatialIndex = buildSpatialIndex(datasetGraph, null); + return spatialIndex; + } + + /** + * Build Spatial Index from all graphs in Dataset.
+ * Dataset contains SpatialIndex in Context. + * + * @param datasetGraph + * @param srsURI + * @return SpatialIndex constructed. + * @throws SpatialIndexException + */ + public static SpatialIndexPerGraph buildSpatialIndex(DatasetGraph datasetGraph, String srsURI) throws SpatialIndexException { + return buildSpatialIndexPerGraph(datasetGraph, srsURI); + } + + public static SpatialIndexPerGraph buildSpatialIndexPerGraph(DatasetGraph datasetGraph, String srsURI) throws SpatialIndexException { + Objects.requireNonNull(datasetGraph); + + if (srsURI == null) { + // XXX Dataset wrapping due to legacy code. + Dataset dataset = DatasetFactory.wrap(datasetGraph); + srsURI = GeoSPARQLOperations.findModeSRS(dataset); + } + + // XXX SpatialIndexerComputation could be adapted to run the code below if just 1 thread is requested. + // SpatialIndexerComputation computation = new SpatialIndexerComputation(datasetGraph, srsURI, null, 1); + + STRtreePerGraph treePerGraph; + logger.info("Building Spatial Index - Started"); + try (AutoTxn txn = Txn.autoTxn(datasetGraph, TxnType.READ)) { + treePerGraph = STRtreeUtils.buildSpatialIndexTree(datasetGraph, srsURI); + txn.commit(); + } + logger.info("Building Spatial Index - Completed"); + + SRSInfo srsInfo = SRSRegistry.getSRSInfo(srsURI); + SpatialIndexPerGraph index = new SpatialIndexPerGraph(srsInfo, treePerGraph, null); + setSpatialIndex(datasetGraph, index); + return index; + } + + public static Node unwrapGraphName(Graph graph) { + Node graphNode = graph instanceof NamedGraph namedGraph + ? namedGraph.getGraphName() + : null; + return graphNode; + } + + public static BasicTask scheduleOnceIndexTask(DatasetGraph dsg, SpatialIndexerComputation indexComputation, Path targetFile, boolean isReplaceTask, + TaskListener taskListener) { + Context cxt = dsg.getContext(); + + BasicTask task = cxt.compute(SpatialIndexConstants.symSpatialIndexTask, (key, priorTaskObj) -> { + BasicTask priorTask = (BasicTask)priorTaskObj; + if (priorTask != null && !priorTask.isTerminated()) { + throw new RuntimeException("A spatial indexing task is already active for this dataset. Wait for completion or abort it."); + } + + TaskThread thread = createIndexerTask(dsg, null, indexComputation, taskListener, targetFile, isReplaceTask); + thread.start(); + return thread; + }); + + return task; + } + + public static TaskThread createIndexerTask(DatasetGraph dsg, Predicate isAuthorizedGraph, SpatialIndexerComputation indexComputation, TaskListener taskListener, Path targetFile, boolean isReplaceTask) { + Context cxt = dsg.getContext(); + long graphCount = indexComputation.getGraphNodes().size(); + boolean isEffectiveUpdate = !isReplaceTask || isAuthorizedGraph != null; + + TaskThread thread = new TaskThread("Spatial Indexer Task", taskListener) { + @Override + public void runActual() throws Exception { + // Prevent deletions of graphs from the index which a user cannot see due to access restrictions. + // For this we need the physical graph list. + // Replace task on a visible subset is an update. + // With removal of all graphs in visible but not in selected. + + if (logger.isInfoEnabled()) { + String replaceMsg = isReplaceTask ? "The resulting index will REPLACE a prior index." : "A prior index will be UPDATED with the newly indexed graphs."; + logger.info("Indexing of {} graphs started. " + replaceMsg, graphCount); + } + + SpatialIndexPerGraph rawOldIndex = SpatialIndexLib.getSpatialIndex(cxt); + + // Check that prior and new SRS are consistent. + if (isEffectiveUpdate) { + String priorSrs = Optional.ofNullable(rawOldIndex) + .map(SpatialIndex::getSrsInfo).map(SRSInfo::getSrsURI).orElse(null); + String requestedSrs = indexComputation.getSrsURI(); + + if (priorSrs != null && !priorSrs.equals(requestedSrs)) { + throw new IllegalArgumentException("The SRS of the update request is inconistent with the SRS of the index: index SRS: " + priorSrs + ", requested SRS: " + requestedSrs); + } + } + + // Uncomment to test artificial delays. + // Thread.sleep(5000); + + SpatialIndexPerGraph newIndex = indexComputation.call(); + + // If NOT in replace mode, add all graph-indexes from the previous index + if (isEffectiveUpdate) { + // Copy the old index into a new one. + SpatialIndexPerGraph oldIndex = (SpatialIndexPerGraph)rawOldIndex; + if (oldIndex != null) { + Map oldTreeMap = oldIndex.getIndex().getTreeMap(); + + oldTreeMap.forEach((name, tree) -> { + boolean isGraphNotSelectedForUpdate = !newIndex.getIndex().contains(name); + + boolean addGraph = false; + if (isReplaceTask) { + boolean isNonAuthorizedGraph = isAuthorizedGraph != null && !isAuthorizedGraph.test(name); + if (isNonAuthorizedGraph) { + addGraph = true; + } + } else { + addGraph = isGraphNotSelectedForUpdate; + } + + if (addGraph) { + newIndex.getIndex().setTree(name, tree); + } + }); + } + } + + SpatialIndexLib.setSpatialIndex(cxt, newIndex); + if (targetFile != null) { + newIndex.setLocation(targetFile); + logger.info("Writing spatial index of {} graphs to disk at path {}", graphCount, targetFile.toAbsolutePath()); + SpatialIndexIoKryo.save(targetFile, newIndex); + } + String statusMsg = String.format("Updated spatial index with %d graphs.", graphCount); + setStatusMessage(statusMsg); + if (logger.isInfoEnabled()) { + logger.info("Indexing of {} graphs completed successfully.", graphCount); + } + } + + @Override + public void requestCancel() { + indexComputation.abort(); + super.requestCancel(); // Interrupt + } + }; + + return thread; + } + + /** + * Attempt to start a spatial index task that cleans the index of graphs not present in the given dataset. + * This method fails if there is already another spatial index task running. + */ + public static BasicTask scheduleOnceCleanTask(DatasetGraph dsg, TaskListener taskListener) { + Context cxt = dsg.getContext(); + BasicTask task = cxt.compute(SpatialIndexConstants.symSpatialIndexTask, (key, priorTaskObj) -> { + BasicTask priorTask = (BasicTask) priorTaskObj; + if (priorTask != null && !priorTask.isTerminated()) { + throw new RuntimeException("A spatial indexing task is already active for this dataset. Wait for completion or abort it."); + } + + TaskThread thread = createCleanTask(dsg, null, taskListener); + thread.start(); + return thread; + }); + return task; + } + + public static TaskThread createCleanTask(DatasetGraph dsg, Predicate isAuthorizedGraph, TaskListener taskListener) { + Context cxt = dsg.getContext(); + + TaskThread thread = new TaskThread("Clean action", taskListener) { + @Override + public void runActual() throws Exception { + SpatialIndex spatialIndexRaw = SpatialIndexLib.getSpatialIndex(cxt); + if (spatialIndexRaw == null) { + throw new SpatialIndexException("No spatial index available on current dataset."); + } else if (spatialIndexRaw instanceof SpatialIndexPerGraph spatialIndex) { + // Prevent deletions of graphs from the index which a user cannot see due to access restrictions. + // For this we need the physical graph list. + Set physicalGraphs = Txn.calculateRead(dsg, () -> accGraphNodes(new LinkedHashSet<>(), dsg)); + + STRtreePerGraph perGraphIndex = spatialIndex.getIndex(); + Map treeMap = perGraphIndex.getTreeMap(); + Set visibleGraphNodes = Txn.calculateRead(dsg, () -> accGraphNodes(new LinkedHashSet<>(), dsg)); + if (physicalGraphs == null) { + physicalGraphs = visibleGraphNodes; + visibleGraphNodes = null; + } + + List indexGraphNodes = new ArrayList<>(treeMap.keySet()); + + int cleanCount = 0; + for (Node node : indexGraphNodes) { + if (!(node == null || Quad.isDefaultGraph(node))) { // Can never delete the default graph. + // A graph is only subject to removal if it is not in the physical graph list. + if (!physicalGraphs.contains(node)) { + // If the graph subject to removal is not visible to the user then reject the removal + if (isAuthorizedGraph != null && !isAuthorizedGraph.test(node)) { + // Prevent removal of a graph we are not authorized to. + continue; + } + perGraphIndex.removeTree(node); + ++cleanCount; + // System.out.println("Removed: [" + node + "] " + (node == null)); + } + } + } + + int finalGraphCount = treeMap.keySet().size(); + Path targetFile = spatialIndex.getLocation(); + // SpatialIndexUtils.setSpatialIndex(cxt, newIndex); + if (cleanCount > 0 && targetFile != null) { + // newIndex.setLocation(targetFile); + logger.info("Writing spatial index of {} graphs (cleaned: {}) to disk at path {}", finalGraphCount, cleanCount, targetFile.toAbsolutePath()); + SpatialIndexIoKryo.save(targetFile, spatialIndex); + } + String statusMsg = String.format("Updated spatial index of %d graphs (cleaned: %d)", finalGraphCount, cleanCount); + setStatusMessage(statusMsg); + logger.info("Indexing of {} graphs completed successfully.", finalGraphCount); + } else { + throw new SpatialIndexException("Unsupported spatial index type for cleaning."); + } + } + }; + return thread; + } + + public static > C accGraphNodes(C accGraphs, DatasetGraph dsg) { + try (Stream s = Iter.asStream(dsg.listGraphNodes())) { + s.forEach(accGraphs::add); + } + return accGraphs; + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexPerGraph.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexPerGraph.java new file mode 100644 index 00000000000..b3f0fbb0b96 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexPerGraph.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.lang.invoke.MethodHandles; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Objects; +import java.util.Set; + +import org.apache.jena.geosparql.implementation.SRSInfo; +import org.apache.jena.geosparql.implementation.registry.SRSRegistry; +import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.query.TxnType; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.system.AutoTxn; +import org.apache.jena.system.Txn; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.index.strtree.STRtree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * SpatialIndex for testing bounding box collisions between geometries within a + * Dataset.
+ * Queries must be performed using the same SRS URI as the SpatialIndex.
+ * The SpatialIndex is added to the Dataset Context when it is built.
+ * QueryRewriteIndex is also stored in the SpatialIndex as its content is + * Dataset specific. + * + */ +public class SpatialIndexPerGraph implements SpatialIndex { + + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private transient final SRSInfo srsInfo; + private STRtreePerGraph index; + private Path location; + + public SpatialIndexPerGraph(STRtreePerGraph index) { + this(SRS_URI.DEFAULT_WKT_CRS84, index, null); + } + + public SpatialIndexPerGraph(String srsUri, STRtreePerGraph index, Path location) { + this( + SRSRegistry.getSRSInfo(srsUri), + index, + null); + } + + public SpatialIndexPerGraph(SRSInfo srsInfo, STRtreePerGraph index, Path location) { + super(); + this.srsInfo = Objects.requireNonNull(srsInfo); + this.index = Objects.requireNonNull(index); + this.location = location; + } + + public STRtreePerGraph getIndex() { + return index; + } + + /** + * + * @return Information about the SRS used by the SpatialIndex. + */ + @Override + public SRSInfo getSrsInfo() { + return srsInfo; + } + + /** + * + * @return True if the SpatialIndex is empty. + */ + @Override + public boolean isEmpty() { + return index.isEmpty(); + } + + /** + * Returns the number of items in the index. + */ + @Override + public long getSize() { + return index.size(); + } + + @Override + public Collection query(Envelope searchEnvelope, Node graph) { + return index.queryOneGraph(searchEnvelope, graph); + } + + @Override + public Path getLocation() { + return location; + } + + @Override + public void setLocation(Path location) { + this.location = location; + } + + @Override + public String toString() { + return "SpatialIndex{" + "srsInfo=" + srsInfo + ", index=" + index + ", file=" + location + '}'; + } + + /** + * Recompute and replace the spatial index trees for the given named graphs. + * + * @param index the spatial index to modify + * @param datasetGraph the dataset containing the named graphs + * @param graphNames the named graphs + * @throws SpatialIndexException + */ + public static void recomputeIndexForGraphs(SpatialIndexPerGraph index, + DatasetGraph datasetGraph, + Set graphNames) throws SpatialIndexException { + STRtreePerGraph trees = index.getIndex(); + try (AutoTxn txn = Txn.autoTxn(datasetGraph, TxnType.READ)) { + for (String graphName : graphNames) { + Node g = graphName == null ? null : NodeFactory.createURI(graphName); + if (trees.contains(g)) { + LOGGER.info("recomputing spatial index for graph: {}", graphName); + } else { + LOGGER.info("computing spatial index for graph: {}", graphName); + } + Graph namedGraph = datasetGraph.getGraph(g); + STRtree indexTree = STRtreeUtils.buildSpatialIndexTree(namedGraph, index.getSrsInfo().getSrsURI()); + STRtree oldIndexTree = trees.setTree(g, indexTree); + if (oldIndexTree != null) { + LOGGER.info("replaced spatial index for graph: {}", graphName); + } else { + LOGGER.info("added spatial index for graph: {}", graphName); + } + } + txn.commit(); + } + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexerComputation.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexerComputation.java new file mode 100644 index 00000000000..ada66757c6c --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexerComputation.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.geosparql.spatial.index.v2; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.query.TxnType; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.sparql.engine.iterator.Abortable; +import org.apache.jena.system.AutoTxn; +import org.apache.jena.system.Txn; +import org.locationtech.jts.index.strtree.STRtree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** Low level class to compute geo indexes for a given set of graphs. */ +public class SpatialIndexerComputation + implements Callable, Abortable +{ + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private DatasetGraph datasetGraph; + private List graphNodes; + private ExecutorService executorService; + private String srsURI; + private int threadCount; + + private final AtomicBoolean requestingCancel = new AtomicBoolean(); + private volatile boolean cancelOnce = false; + private Object cancelLock = new Object(); + + private List>> futures = new ArrayList<>(); + + private static boolean logProgress = false; + + /** + * Create an instance of a spatial indexer computation. + * + * @param datasetGraph The dataset graph for which to index the spatial data. + * @param srsURI The spatial reference system against which to index the data. + * @param graphNodes The names of the physical graphs which to process. May use Quad.defaultGraphIRI but not e.g. Quad.unionGraph. + * @param threadCount Maximum number of threads to use for indexing. Must be at least 1. + */ + public SpatialIndexerComputation(DatasetGraph datasetGraph, String srsURI, List graphNodes, int threadCount) { + this.datasetGraph = datasetGraph; + this.graphNodes = graphNodes; + this.srsURI = srsURI; + this.threadCount = threadCount; + + if (threadCount <= 0) { + throw new IllegalArgumentException("Thread count must be greater than 0."); + } + } + + public String getSrsURI() { + return srsURI; + } + + public List getGraphNodes() { + return graphNodes; + } + + private Entry indexOneGraph(Node graphNode) throws SpatialIndexException { + if (logProgress) { + LOGGER.info("Started spatial index build for graph {} ...", graphNode); + } + + STRtree tree = null; + + try (AutoTxn txn = Txn.autoTxn(datasetGraph, TxnType.READ)) { + Graph graph = Quad.isDefaultGraph(graphNode) // XXX would getGraph work with the default graph URIs? + ? datasetGraph.getDefaultGraph() + : datasetGraph.getGraph(graphNode); + if (graph != null) { // May be null if the requested graph does not exist (possibly due to a dynamic dataset) + tree = STRtreeUtils.buildSpatialIndexTree(graph, srsURI); + } + + // XXX This commit is a workaround for DatasetGraphText.abort() causing a NPE in + // during multi-threaded spatial index computation. + // This is an issue related to the transaction mechanics of DatasetGraphText. + txn.commit(); + } + + if (logProgress) { + LOGGER.info("Completed spatial index for graph {}", graphNode); + } + return Map.entry(graphNode, tree); + } + + /** Returns a {@link SpatialIndexPerGraph} instance with the configured SRS. */ + @Override + public SpatialIndexPerGraph call() throws InterruptedException, ExecutionException { + try { + return callActual(); + } finally { + cleanUp(); + } + } + + protected SpatialIndexPerGraph callActual() throws InterruptedException, ExecutionException { + synchronized (cancelLock) { + if (executorService != null) { + throw new IllegalStateException("Task already running."); + } + executorService = Executors.newFixedThreadPool(threadCount); + } + + // Beware: We expect special graphNodes such as Quad.unionGraph to have been resolved + // to the explicit graph names before coming here. + for (Node graphNode : graphNodes) { + checkRequestingCancel(); + Future> future = executorService.submit(() -> indexOneGraph(graphNode)); + futures.add(future); + } + + // Collect all futures into a map. + Map treeMap = new LinkedHashMap(); + for (Future> future : futures) { + Entry entry = future.get(); + Node graphNode = entry.getKey(); + STRtree tree = entry.getValue(); + if (tree != null) { + treeMap.put(graphNode, tree); + } + } + + STRtreePerGraph trees = new STRtreePerGraph(); + trees.setTrees(treeMap); + SpatialIndexPerGraph result = new SpatialIndexPerGraph(trees); + return result; + } + + protected void cleanUp() { + if (executorService != null) { + executorService.shutdownNow(); + try { + executorService.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.warn("Abandoning an executor service that failed to stop.", e); + } + } + } + + protected void checkRequestingCancel() { + boolean isCancelled = requestingCancel(); + if (isCancelled) { + throw new CancellationException(); + } + } + + @Override + public void abort() { + synchronized (cancelLock) { + requestingCancel.set(true); + if (!cancelOnce) { + requestCancel(); + cancelOnce = true; + } + } + } + + protected void requestCancel() { } + + private boolean requestingCancel() { + return (requestingCancel != null && requestingCancel.get()) || Thread.interrupted() ; + } + + protected void performRequestingCancel() { + // Tasks will be aborted on close - so nothing to do here. + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunction.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunction.java index aa1b2e3427e..12625b59acc 100644 --- a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunction.java +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunction.java @@ -18,7 +18,7 @@ package org.apache.jena.geosparql.spatial.property_functions; import java.util.Arrays; -import java.util.HashSet; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.stream.Stream; @@ -33,10 +33,10 @@ import org.apache.jena.geosparql.spatial.SearchEnvelope; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; import org.apache.jena.graph.Triple; -import org.apache.jena.rdf.model.Resource; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.ExecutionContext; import org.apache.jena.sparql.engine.QueryIterator; @@ -66,7 +66,7 @@ public abstract class GenericSpatialPropertyFunction extends PFuncSimpleAndList @Override public final QueryIterator execEvaluated(Binding binding, Node subject, Node predicate, PropFuncArg object, ExecutionContext execCxt) { try { - spatialIndex = SpatialIndex.retrieve(execCxt); + spatialIndex = SpatialIndexLib.retrieve(execCxt); spatialArguments = extractObjectArguments(predicate, object, spatialIndex.getSrsInfo()); return search(binding, execCxt, subject, spatialArguments.limit); } catch (SpatialIndexException ex) { @@ -184,15 +184,17 @@ private QueryIterator checkUnbound(Binding binding, ExecutionContext execCxt, No //Find all Features in the spatial index which are within the rough search envelope. SearchEnvelope searchEnvelope = spatialArguments.searchEnvelope; - HashSet features = searchEnvelope.check(spatialIndex); + Graph activeGraph = execCxt.getActiveGraph(); + Node graphName = SpatialIndexLib.unwrapGraphName(activeGraph); + Collection features = searchEnvelope.check(spatialIndex, graphName); Var subjectVar = Var.alloc(subject.getName()); - Stream stream = features.stream(); + Stream stream = features.stream(); if (requireSecondFilter()) { - stream = stream.filter(feature -> checkBound(execCxt, feature.asNode())); + stream = stream.filter(feature -> checkBound(execCxt, feature)); } - Iterator iterator = stream.map(feature -> BindingFactory.binding(binding, subjectVar, feature.asNode())) + Iterator iterator = stream.map(feature -> BindingFactory.binding(binding, subjectVar, feature)) .limit(limit) .iterator(); return QueryIterPlainWrapper.create(iterator, execCxt); diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/BasicTask.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/BasicTask.java new file mode 100644 index 00000000000..496f9e153cd --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/BasicTask.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.geosparql.spatial.task; + +import org.apache.jena.sparql.engine.iterator.Abortable; + +/** An outside view of a running task */ +public interface BasicTask extends Abortable { + + public interface TaskListener { + void onStateChange(T task); + } + + TaskState getTaskState(); + + /** A label for the task. */ + String getLabel(); + + @Override + void abort(); + + long getCreationTime(); + long getStartTime(); + long getEndTime(); + long getAbortTime(); + + /** If non null, the throwable that is the cause for an exceptional termination of the task. */ + Throwable getThrowable(); + + /** Get the last status message of the task. May be null. */ + String getStatusMessage(); + + /** Whether abort has been called. */ + // XXX this might be different from whether the task actually transitioned into aborting state. + boolean isAborting(); + + default boolean isTerminated() { + TaskState state = getTaskState(); + return TaskState.TERMINATED.equals(state); + } +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/TaskState.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/TaskState.java new file mode 100644 index 00000000000..add2ab9e086 --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/TaskState.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.geosparql.spatial.task; + +public enum TaskState { + CREATED, // Task object created. + STARTING, // Task execution requested, but core task.run() method not yet invoked. + RUNNING, // Core task.run() method invoked. + ABORTING, // Abort called while not in TERMINATING / TERMINATED state. + TERMINATING, // Core task.run() method exited. + TERMINATED, // Task cleanup complete. Triggering this event may require a call to close(). +} diff --git a/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/TaskThread.java b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/TaskThread.java new file mode 100644 index 00000000000..519778ea58d --- /dev/null +++ b/jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/TaskThread.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.geosparql.spatial.task; + +import java.util.Objects; +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Thread base class that provides a {@link #abort()} method that can run a custom action + * (in addition to setting the interrupted flag) as well as a doAfterRun() method + * for cleaning up after execution (only gets called if there is a prior call to run). + */ +public abstract class TaskThread + extends Thread + implements BasicTask +{ + private static final Logger logger = LoggerFactory.getLogger(TaskThread.class); + + private final AtomicBoolean requestingCancel; + private Object cancelLock = new Object(); + + private TaskState state = TaskState.CREATED; + private TaskListener taskListener; + + // protected List> completionHandlers = new ArrayList<>(); + + private boolean isComplete = false; + private Throwable throwable; + + private String label; + private String statusMessage; + + /** + * Time of the first abort request; null if there is none. + * Cancel time will not be set if cancel is called after a task's completion. + */ + private long creationTime = -1; + private long startTime = -1; // When when run() is called. + private long cancelTime = -1; + private long endTime = -1; // Time when runActual returns - but before afterRun() is called. + + // XXX Add native support for CompletableFuture? + + private void updateState(TaskState state) { + synchronized (cancelLock) { + this.state = state; + if (taskListener != null) { + try { + taskListener.onStateChange(this); + } catch (Throwable t) { + logger.warn("Listener raised an exception.", t); + } + } + } + } + + public TaskThread(String label, TaskListener taskListener) { + this(label, taskListener, new AtomicBoolean()); + } + + public TaskThread(String label, TaskListener taskListener, AtomicBoolean requestingCancel) { + super(); + this.label = label; + this.taskListener = taskListener; + this.requestingCancel = Objects.requireNonNull(requestingCancel); + + this.creationTime = System.currentTimeMillis(); + updateState(TaskState.CREATED); + } + + @Override + public String getLabel() { + return label; + } + + @Override + public TaskState getTaskState() { + return state; + } + + @Override + public Throwable getThrowable() { + return throwable; + } + + @Override + public long getCreationTime() { + return creationTime; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public long getEndTime() { + return endTime; + } + + @Override + public long getAbortTime() { + return cancelTime; + } + + protected void setStatusMessage(String statusMessage) { + this.statusMessage = statusMessage; + } + + @Override + public String getStatusMessage() { + return statusMessage; + } + + protected void beforeRun() { + } + + @Override + public final void run() { + synchronized (cancelLock) { + if (!TaskState.CREATED.equals(state)) { + // Task may have already been started or aborted. + throw new IllegalStateException("Can only start tasks in CREATED state, but this task is in state: " + state); + } + + startTime = System.currentTimeMillis(); + updateState(TaskState.STARTING); + } + + runInternal(); + } + + public final void runInternal() { + try { + checkCancelled(); + beforeRun(); + + checkCancelled(); + runActual(); + } catch (Throwable t) { + throwable = t; + updateState(TaskState.TERMINATING); + throw new RuntimeException(t); + } finally { + endTime = System.currentTimeMillis(); + isComplete = true; + try { + doAfterRun(); + } finally { + updateState(TaskState.TERMINATED); + } + } + } + + /** The actual "run" method that must be implemented. */ + public abstract void runActual() throws Exception; + + protected void requestCancel() { + // If the task has not been started, then directly transition to terminated with a cancellation exception. + // Otherwise transition to aborting state. + if (TaskState.CREATED.equals(state)) { + this.throwable = new CancellationException(); + updateState(TaskState.TERMINATED); + } else { + updateState(TaskState.ABORTING); + this.interrupt(); + } + } + + protected void doAfterRun() { + } + + /** Raises a {@link CancellationException} if abort was called. */ + protected void checkCancelled() { + if (isAborting()) { + throw new CancellationException(); + } + } + + @Override + public void abort() { + synchronized (cancelLock) { + if ( ! requestingCancel.get() ) { + requestingCancel.set(true); + cancelTime = System.currentTimeMillis(); + if (!isComplete) { + this.requestCancel(); + } + } + } + } + + @Override + public boolean isAborting() { + return requestingCancel.get(); + } + + @Override + public boolean isTerminated() { + return !isAlive(); + } +} diff --git a/jena-geosparql/src/main/java/org/locationtech/jts/index/strtree/STRtreeSerializer.java b/jena-geosparql/src/main/java/org/locationtech/jts/index/strtree/STRtreeSerializer.java new file mode 100644 index 00000000000..127a2f8e9c6 --- /dev/null +++ b/jena-geosparql/src/main/java/org/locationtech/jts/index/strtree/STRtreeSerializer.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.locationtech.jts.index.strtree; + +import java.util.ArrayList; +import java.util.List; + +import org.locationtech.jts.geom.Envelope; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +/* + * This file is an adapted copy of org.locationtech.jts.index.strtree.IndexSerde from + * the Apache Sedona project: + * + * https://github.com/apache/sedona/blob/sedona-1.7.0/common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java + * + * This file contains the class STRtreeSerializer for kryo-based serialization of + * STRtree instances from the Java Topology Suite (JTS). + * + * The STRtreeSerializer must remain in the org.locationtech.jts.index.strtree package because + * it accesses non-public members of STRtree. + * + * The change over the Sedona version is, that the tree content is written out by handing control + * idiomatically back to kryo instead of making assumptions about the tree content. + */ + +/** + * Provides methods to efficiently serialize and deserialize the index. trees + * are serialized recursively. + */ +@SuppressWarnings({ "rawtypes", "unchecked" }) +public class STRtreeSerializer + extends Serializer +{ + @Override + public STRtree read(Kryo kryo, Input input, Class type) { + int nodeCapacity = input.readInt(); + boolean notEmpty = (input.readByte() & 0x01) == 1; + if (notEmpty) { + boolean built = (input.readByte() & 0x01) == 1; + if (built) { + // if built, root is not null, set itemBoundables to null + STRtree index = new STRtree(nodeCapacity, readSTRtreeNode(kryo, input)); + return index; + } else { + // if not built, just read itemBoundables + ArrayList itemBoundables = new ArrayList(); + int itemSize = input.readInt(); + for (int i = 0; i < itemSize; ++i) { + itemBoundables.add(readItemBoundable(kryo, input)); + } + STRtree index = new STRtree(nodeCapacity, itemBoundables); + return index; + } + } else { + return new STRtree(nodeCapacity); + } + } + + @Override + public void write(Kryo kryo, Output output, STRtree tree) { + output.writeInt(tree.getNodeCapacity()); + if (tree.isEmpty()) { + output.writeByte(0); + } else { + output.writeByte(1); + // write head + boolean isBuilt = tree.getItemBoundables() == null; + output.writeByte(isBuilt ? 1 : 0); + if (!isBuilt) { + // if not built, itemBoundables will not be null, record it + ArrayList itemBoundables = tree.getItemBoundables(); + output.writeInt(itemBoundables.size()); + for (Object obj : itemBoundables) { + if (!(obj instanceof ItemBoundable)) { + throw new UnsupportedOperationException( + " itemBoundables should only contain ItemBoundable objects "); + } + ItemBoundable itemBoundable = (ItemBoundable) obj; + // write envelope + writeItemBoundable(kryo, output, itemBoundable); + } + } else { + // if built, write from root + writeSTRTreeNode(kryo, output, tree.getRoot()); + } + } + } + + private void writeSTRTreeNode(Kryo kryo, Output output, AbstractNode node) { + // write head + output.writeInt(node.getLevel()); + // write children + List children = node.getChildBoundables(); + int childrenSize = children.size(); + output.writeInt(childrenSize); + // if children not empty, write children + if (childrenSize > 0) { + if (children.get(0) instanceof AbstractNode) { + // write type as 0, non-leaf node + output.writeByte(0); + for (Object obj : children) { + AbstractNode child = (AbstractNode) obj; + writeSTRTreeNode(kryo, output, child); + } + } else if (children.get(0) instanceof ItemBoundable) { + // write type as 1, leaf node + output.writeByte(1); + // for leaf node, write items + for (Object obj : children) { + writeItemBoundable(kryo, output, (ItemBoundable) obj); + } + } else { + throw new UnsupportedOperationException("wrong node type of STRtree"); + } + } + } + + private STRtree.STRtreeNode readSTRtreeNode(Kryo kryo, Input input) { + int level = input.readInt(); + STRtree.STRtreeNode node = new STRtree.STRtreeNode(level); + int childrenSize = input.readInt(); + boolean isLeaf = (input.readByte() & 0x01) == 1; + ArrayList children = new ArrayList(); + if (isLeaf) { + for (int i = 0; i < childrenSize; ++i) { + children.add(readItemBoundable(kryo, input)); + } + } else { + for (int i = 0; i < childrenSize; ++i) { + children.add(readSTRtreeNode(kryo, input)); + } + } + node.setChildBoundables(children); + return node; + } + + private void writeItemBoundable(Kryo kryo, Output output, ItemBoundable itemBoundable) { + kryo.writeObject(output, itemBoundable.getBounds()); + kryo.writeClassAndObject(output, itemBoundable.getItem()); + } + + private ItemBoundable readItemBoundable(Kryo kryo, Input input) { + Envelope envelope = kryo.readObject(input, Envelope.class); + Object item = kryo.readClassAndObject(input); + return new ItemBoundable(envelope, item); + } +} diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/CancelQueryTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/CancelQueryTest.java index 7b54ebb9049..ccb0d8d4194 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/CancelQueryTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/CancelQueryTest.java @@ -33,6 +33,7 @@ import org.apache.jena.geosparql.implementation.vocabulary.Geo; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; import org.apache.jena.graph.Graph; import org.apache.jena.graph.NodeFactory; import org.apache.jena.query.Dataset; @@ -106,8 +107,8 @@ public void test_cancel_spatial_property_function1() { // create spatial index if (useIndex){ try { - SpatialIndex index = SpatialIndex.buildSpatialIndex(ds); - SpatialIndex.setSpatialIndex(ds, index); + SpatialIndex index = SpatialIndexLib.buildSpatialIndex(ds.asDatasetGraph()); + SpatialIndexLib.setSpatialIndex(ds, index); } catch (SpatialIndexException e) { throw new RuntimeException(e); } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunctionTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunctionTest.java index 5a99718258f..fd405c1b770 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunctionTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/GenericPropertyFunctionTest.java @@ -17,7 +17,17 @@ */ package org.apache.jena.geosparql.geo.topological; -import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.*; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.FEATURE_A; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.FEATURE_B; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.FEATURE_D; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEOMETRY_A; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEOMETRY_B; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEOMETRY_C_BLANK; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEOMETRY_D; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEOMETRY_F; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEO_FEATURE_Y; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEO_FEATURE_Z; +import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.TEST_SRS_URI; import static org.junit.Assert.assertEquals; import java.util.ArrayList; @@ -30,16 +40,25 @@ import org.apache.jena.geosparql.implementation.index.IndexConfiguration.IndexOption; import org.apache.jena.geosparql.implementation.index.QueryRewriteIndex; import org.apache.jena.geosparql.implementation.vocabulary.Geo; -import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; import org.apache.jena.graph.NodeFactory; -import org.apache.jena.query.*; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryExecutionFactory; +import org.apache.jena.query.QuerySolution; +import org.apache.jena.query.ResultSet; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; import org.apache.jena.vocabulary.RDF; -import org.junit.*; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; /** * @@ -47,8 +66,11 @@ */ public class GenericPropertyFunctionTest { - private static Model model; - private static Dataset dataset; + private static Model modelGeoSparql; + private static Dataset datasetGeoSparql; + + private static Model modelWgs84; + private static Dataset datasetWgs84; public GenericPropertyFunctionTest() { } @@ -56,8 +78,11 @@ public GenericPropertyFunctionTest() { @BeforeClass public static void setUpClass() throws SpatialIndexException { GeoSPARQLConfig.setup(IndexOption.MEMORY, Boolean.TRUE); - model = QueryRewriteTestData.createTestData(); - dataset = SpatialIndex.wrapModel(model, TEST_SRS_URI); + modelGeoSparql = QueryRewriteTestData.addTestDataGeoSparql(ModelFactory.createDefaultModel()); + datasetGeoSparql = SpatialIndexLib.wrapModel(modelGeoSparql, TEST_SRS_URI); + + modelWgs84 = QueryRewriteTestData.addTestDataWgs84(ModelFactory.createDefaultModel()); + datasetWgs84 = SpatialIndexLib.wrapModel(modelWgs84, TEST_SRS_URI); } @AfterClass @@ -78,7 +103,7 @@ public void tearDown() { @Test public void testQueryRewrite_geometry_geometry() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = GEOMETRY_B.asNode(); @@ -95,7 +120,7 @@ public void testQueryRewrite_geometry_geometry() { @Test public void testQueryRewrite_geometry_geometry_blank() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = GEOMETRY_C_BLANK.asNode(); @@ -112,8 +137,7 @@ public void testQueryRewrite_geometry_geometry_blank() { @Test public void testQueryRewrite_blank() { - - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Boolean expResult = true; String id = GEOMETRY_C_BLANK.asNode().getBlankNodeLabel(); @@ -129,7 +153,7 @@ public void testQueryRewrite_blank() { @Test public void testQueryRewrite_geometry_geometry_disabled() { GeoSPARQLConfig.setup(IndexOption.MEMORY, Boolean.FALSE); - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = FEATURE_B.asNode(); @@ -148,7 +172,7 @@ public void testQueryRewrite_geometry_geometry_disabled() { @Test public void testQueryRewrite_feature_geometry() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = FEATURE_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = GEOMETRY_B.asNode(); @@ -165,7 +189,7 @@ public void testQueryRewrite_feature_geometry() { @Test public void testQueryRewrite_feature_feature() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = FEATURE_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = FEATURE_B.asNode(); @@ -182,7 +206,7 @@ public void testQueryRewrite_feature_feature() { @Test public void testQueryRewrite_geometry_feature() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = FEATURE_B.asNode(); @@ -200,7 +224,7 @@ public void testQueryRewrite_geometry_feature() { public void testQueryRewrite_geometry_feature_disabled() { GeoSPARQLConfig.setup(IndexOption.MEMORY, Boolean.FALSE); - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = FEATURE_B.asNode(); @@ -219,7 +243,7 @@ public void testQueryRewrite_geometry_feature_disabled() { @Test public void testQueryRewrite_geometry_geometry_false() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = GEOMETRY_D.asNode(); @@ -236,7 +260,7 @@ public void testQueryRewrite_geometry_geometry_false() { @Test public void testQueryRewrite_feature_geometry_false() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = FEATURE_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = GEOMETRY_D.asNode(); @@ -253,7 +277,7 @@ public void testQueryRewrite_feature_geometry_false() { @Test public void testQueryRewrite_feature_feature_false() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = FEATURE_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = FEATURE_D.asNode(); @@ -270,7 +294,7 @@ public void testQueryRewrite_feature_feature_false() { @Test public void testQueryRewrite_geometry_feature_false() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = FEATURE_D.asNode(); @@ -288,7 +312,7 @@ public void testQueryRewrite_geometry_feature_false() { public void testQueryRewrite_geometry_geometry_asserted() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = GEOMETRY_F.asNode(); @@ -306,7 +330,7 @@ public void testQueryRewrite_geometry_geometry_asserted() { public void testQueryRewrite_geometry_geometry_asserted_disabled() { GeoSPARQLConfig.setup(IndexOption.MEMORY, Boolean.FALSE); - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = GEOMETRY_A.asNode(); Node predicate = Geo.SF_CONTAINS_NODE; Node object = GEOMETRY_F.asNode(); @@ -325,7 +349,7 @@ public void testQueryRewrite_geometry_geometry_asserted_disabled() { @Test public void testQueryRewrite_feature_feature_disjoint() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = FEATURE_A.asNode(); Node predicate = Geo.SF_DISJOINT_NODE; Node object = FEATURE_D.asNode(); @@ -342,7 +366,7 @@ public void testQueryRewrite_feature_feature_disjoint() { @Test public void testQueryRewrite_feature_feature_disjoint_false() { - Graph graph = model.getGraph(); + Graph graph = modelGeoSparql.getGraph(); Node subject = FEATURE_A.asNode(); Node predicate = Geo.SF_DISJOINT_NODE; Node object = FEATURE_B.asNode(); @@ -369,7 +393,7 @@ public void testExecEvaluated_unbound() { List subjects = new ArrayList<>(); List objects = new ArrayList<>(); - try (QueryExecution qe = QueryExecutionFactory.create(query, dataset)) { + try (QueryExecution qe = QueryExecutionFactory.create(query, datasetGeoSparql)) { ResultSet rs = qe.execSelect(); while (rs.hasNext()) { QuerySolution qs = rs.nextSolution(); @@ -405,7 +429,7 @@ public void testExecEvaluated_subject_bound() { + "}ORDER by ?obj"; List objects = new ArrayList<>(); - try (QueryExecution qe = QueryExecutionFactory.create(query, dataset)) { + try (QueryExecution qe = QueryExecutionFactory.create(query, datasetGeoSparql)) { ResultSet rs = qe.execSelect(); while (rs.hasNext()) { QuerySolution qs = rs.nextSolution(); @@ -438,7 +462,7 @@ public void testExecEvaluated_subject_bound_geometry() { + "}ORDER by ?obj"; List objects = new ArrayList<>(); - try (QueryExecution qe = QueryExecutionFactory.create(query, dataset)) { + try (QueryExecution qe = QueryExecutionFactory.create(query, datasetGeoSparql)) { ResultSet rs = qe.execSelect(); while (rs.hasNext()) { QuerySolution qs = rs.nextSolution(); @@ -474,7 +498,7 @@ public void testExecEvaluated_object_bound() { + "}ORDER by ?subj"; List results = new ArrayList<>(); - try (QueryExecution qe = QueryExecutionFactory.create(query, dataset)) { + try (QueryExecution qe = QueryExecutionFactory.create(query, datasetGeoSparql)) { ResultSet rs = qe.execSelect(); while (rs.hasNext()) { QuerySolution qs = rs.nextSolution(); @@ -509,7 +533,7 @@ public void testExecEvaluated_both_bound() { List subjects = new ArrayList<>(); List objects = new ArrayList<>(); - try (QueryExecution qe = QueryExecutionFactory.create(query, dataset)) { + try (QueryExecution qe = QueryExecutionFactory.create(query, datasetGeoSparql)) { ResultSet rs = qe.execSelect(); while (rs.hasNext()) { QuerySolution qs = rs.nextSolution(); @@ -530,11 +554,10 @@ public void testExecEvaluated_both_bound() { /** * Test of execEvaluated method, of class GenericPropertyFunction. + * @throws SpatialIndexException */ @Test - public void testExecEvaluated_both_bound_geo() { - - + public void testExecEvaluated_both_bound_geo() throws SpatialIndexException { String query = "PREFIX geo: \n" + "\n" + "SELECT ?subj ?obj\n" @@ -546,7 +569,7 @@ public void testExecEvaluated_both_bound_geo() { List subjects = new ArrayList<>(); List objects = new ArrayList<>(); - try (QueryExecution qe = QueryExecutionFactory.create(query, dataset)) { + try (QueryExecution qe = QueryExecutionFactory.create(query, datasetWgs84)) { ResultSet rs = qe.execSelect(); while (rs.hasNext()) { QuerySolution qs = rs.nextSolution(); @@ -582,7 +605,7 @@ public void testExecEvaluated_object_is_literal() { + "}ORDER by ?subj"; List results = new ArrayList<>(); - try (QueryExecution qe = QueryExecutionFactory.create(query, dataset)) { + try (QueryExecution qe = QueryExecutionFactory.create(query, datasetGeoSparql)) { ResultSet rs = qe.execSelect(); while (rs.hasNext()) { QuerySolution qs = rs.nextSolution(); diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/QueryRewriteTestData.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/QueryRewriteTestData.java index a775c7579a6..ce260c70330 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/QueryRewriteTestData.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/QueryRewriteTestData.java @@ -24,7 +24,6 @@ import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; import org.apache.jena.rdf.model.Literal; import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.ResourceFactory; import org.apache.jena.vocabulary.RDF; @@ -56,62 +55,58 @@ public class QueryRewriteTestData { public static final String TEST_SRS_URI = SRS_URI.OSGB36_CRS; - private static Model TEST_MODEL = null; - - public static final Model createTestData() { - - if (TEST_MODEL == null) { - TEST_MODEL = ModelFactory.createDefaultModel(); - - //Geometry - TEST_MODEL.add(GEOMETRY_A, Geo.HAS_SERIALIZATION_PROP, ResourceFactory.createTypedLiteral(" POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))", WKTDatatype.INSTANCE)); - TEST_MODEL.add(GEOMETRY_B, Geo.HAS_SERIALIZATION_PROP, LITERAL_B); - TEST_MODEL.add(GEOMETRY_C_BLANK, Geo.HAS_SERIALIZATION_PROP, ResourceFactory.createTypedLiteral(" POINT(5 5)", WKTDatatype.INSTANCE)); - TEST_MODEL.add(GEOMETRY_D, Geo.HAS_SERIALIZATION_PROP, ResourceFactory.createTypedLiteral(" POINT(11 11)", WKTDatatype.INSTANCE)); - TEST_MODEL.add(GEOMETRY_A, RDF.type, Geo.GEOMETRY_RES); - TEST_MODEL.add(GEOMETRY_B, RDF.type, Geo.GEOMETRY_RES); - TEST_MODEL.add(GEOMETRY_C_BLANK, RDF.type, Geo.GEOMETRY_RES); - TEST_MODEL.add(GEOMETRY_D, RDF.type, Geo.GEOMETRY_RES); - TEST_MODEL.add(GEOMETRY_E, RDF.type, Geo.GEOMETRY_RES); - TEST_MODEL.add(GEOMETRY_F, RDF.type, Geo.GEOMETRY_RES); - - //Feature - TEST_MODEL.add(FEATURE_A, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_A); - TEST_MODEL.add(FEATURE_B, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_B); - TEST_MODEL.add(FEATURE_C, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_C_BLANK); - TEST_MODEL.add(FEATURE_D, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_D); - TEST_MODEL.add(FEATURE_A, Geo.HAS_GEOMETRY_PROP, GEOMETRY_A); - TEST_MODEL.add(FEATURE_B, Geo.HAS_GEOMETRY_PROP, GEOMETRY_B); - TEST_MODEL.add(FEATURE_C, Geo.HAS_GEOMETRY_PROP, GEOMETRY_C_BLANK); - TEST_MODEL.add(FEATURE_D, Geo.HAS_GEOMETRY_PROP, GEOMETRY_D); - TEST_MODEL.add(FEATURE_A, RDF.type, Geo.FEATURE_RES); - TEST_MODEL.add(FEATURE_B, RDF.type, Geo.FEATURE_RES); - TEST_MODEL.add(FEATURE_C, RDF.type, Geo.FEATURE_RES); - TEST_MODEL.add(FEATURE_D, RDF.type, Geo.FEATURE_RES); - - //Spatial Objects - TEST_MODEL.add(FEATURE_A, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(FEATURE_B, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(FEATURE_C, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(FEATURE_D, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(GEOMETRY_A, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(GEOMETRY_B, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(GEOMETRY_C_BLANK, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(GEOMETRY_D, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(GEOMETRY_E, RDF.type, Geo.SPATIAL_OBJECT_RES); - TEST_MODEL.add(GEOMETRY_F, RDF.type, Geo.SPATIAL_OBJECT_RES); - - //Contains asserted - TEST_MODEL.add(GEOMETRY_A, Geo.SF_CONTAINS_PROP, GEOMETRY_F); + public static final Model addTestDataGeoSparql(Model model) { + //Geometry + model.add(GEOMETRY_A, Geo.HAS_SERIALIZATION_PROP, ResourceFactory.createTypedLiteral(" POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))", WKTDatatype.INSTANCE)); + model.add(GEOMETRY_B, Geo.HAS_SERIALIZATION_PROP, LITERAL_B); + model.add(GEOMETRY_C_BLANK, Geo.HAS_SERIALIZATION_PROP, ResourceFactory.createTypedLiteral(" POINT(5 5)", WKTDatatype.INSTANCE)); + model.add(GEOMETRY_D, Geo.HAS_SERIALIZATION_PROP, ResourceFactory.createTypedLiteral(" POINT(11 11)", WKTDatatype.INSTANCE)); + model.add(GEOMETRY_A, RDF.type, Geo.GEOMETRY_RES); + model.add(GEOMETRY_B, RDF.type, Geo.GEOMETRY_RES); + model.add(GEOMETRY_C_BLANK, RDF.type, Geo.GEOMETRY_RES); + model.add(GEOMETRY_D, RDF.type, Geo.GEOMETRY_RES); + model.add(GEOMETRY_E, RDF.type, Geo.GEOMETRY_RES); + model.add(GEOMETRY_F, RDF.type, Geo.GEOMETRY_RES); + + //Feature + model.add(FEATURE_A, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_A); + model.add(FEATURE_B, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_B); + model.add(FEATURE_C, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_C_BLANK); + model.add(FEATURE_D, Geo.HAS_DEFAULT_GEOMETRY_PROP, GEOMETRY_D); + model.add(FEATURE_A, Geo.HAS_GEOMETRY_PROP, GEOMETRY_A); + model.add(FEATURE_B, Geo.HAS_GEOMETRY_PROP, GEOMETRY_B); + model.add(FEATURE_C, Geo.HAS_GEOMETRY_PROP, GEOMETRY_C_BLANK); + model.add(FEATURE_D, Geo.HAS_GEOMETRY_PROP, GEOMETRY_D); + model.add(FEATURE_A, RDF.type, Geo.FEATURE_RES); + model.add(FEATURE_B, RDF.type, Geo.FEATURE_RES); + model.add(FEATURE_C, RDF.type, Geo.FEATURE_RES); + model.add(FEATURE_D, RDF.type, Geo.FEATURE_RES); + + //Spatial Objects + model.add(FEATURE_A, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(FEATURE_B, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(FEATURE_C, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(FEATURE_D, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(GEOMETRY_A, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(GEOMETRY_B, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(GEOMETRY_C_BLANK, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(GEOMETRY_D, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(GEOMETRY_E, RDF.type, Geo.SPATIAL_OBJECT_RES); + model.add(GEOMETRY_F, RDF.type, Geo.SPATIAL_OBJECT_RES); + + //Contains asserted + model.add(GEOMETRY_A, Geo.SF_CONTAINS_PROP, GEOMETRY_F); + + return model; + } - //Geo Features - TEST_MODEL.add(GEO_FEATURE_Y, SpatialExtension.GEO_LAT_PROP, GEO_FEATURE_LAT); - TEST_MODEL.add(GEO_FEATURE_Y, SpatialExtension.GEO_LON_PROP, GEO_FEATURE_LON); - TEST_MODEL.add(GEO_FEATURE_Z, SpatialExtension.GEO_LAT_PROP, GEO_FEATURE_LAT); - TEST_MODEL.add(GEO_FEATURE_Z, SpatialExtension.GEO_LON_PROP, GEO_FEATURE_LON); + public static final Model addTestDataWgs84(Model model) { + //Geo Features + model.add(GEO_FEATURE_Y, SpatialExtension.GEO_LAT_PROP, GEO_FEATURE_LAT); + model.add(GEO_FEATURE_Y, SpatialExtension.GEO_LON_PROP, GEO_FEATURE_LON); + model.add(GEO_FEATURE_Z, SpatialExtension.GEO_LAT_PROP, GEO_FEATURE_LAT); + model.add(GEO_FEATURE_Z, SpatialExtension.GEO_LON_PROP, GEO_FEATURE_LON); - } - return TEST_MODEL; + return model; } - } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteralTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteralTest.java index 24aa748eb61..d4b9e29f562 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteralTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/geo/topological/SpatialObjectGeometryLiteralTest.java @@ -17,13 +17,15 @@ */ package org.apache.jena.geosparql.geo.topological; -import org.apache.jena.datatypes.DatatypeFormatException; -import org.apache.jena.datatypes.xsd.XSDDatatype; import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.FEATURE_B; import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEOMETRY_B; import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEO_FEATURE_LITERAL; import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.GEO_FEATURE_Y; import static org.apache.jena.geosparql.geo.topological.QueryRewriteTestData.LITERAL_B; +import static org.junit.Assert.assertEquals; + +import org.apache.jena.datatypes.DatatypeFormatException; +import org.apache.jena.datatypes.xsd.XSDDatatype; import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; @@ -33,7 +35,6 @@ import org.apache.jena.rdf.model.ResourceFactory; import org.junit.After; import org.junit.AfterClass; -import static org.junit.Assert.*; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -44,14 +45,16 @@ */ public class SpatialObjectGeometryLiteralTest { - private static Model MODEL; + private static Model MODEL_GEOSPARQL; + private static Model MODEL_WGS84; public SpatialObjectGeometryLiteralTest() { } @BeforeClass public static void setUpClass() { - MODEL = QueryRewriteTestData.createTestData(); + MODEL_GEOSPARQL = QueryRewriteTestData.addTestDataGeoSparql(ModelFactory.createDefaultModel()); + MODEL_WGS84 = QueryRewriteTestData.addTestDataWgs84(ModelFactory.createDefaultModel()); } @AfterClass @@ -72,7 +75,7 @@ public void tearDown() { @Test public void testRetrieve() { - Graph graph = MODEL.getGraph(); + Graph graph = MODEL_GEOSPARQL.getGraph(); Node targetSpatialObject = null; SpatialObjectGeometryLiteral instance = SpatialObjectGeometryLiteral.retrieve(graph, targetSpatialObject); @@ -87,7 +90,7 @@ public void testRetrieve() { @Test public void testRetrieveGeometryLiteral_geometry() { - Graph graph = MODEL.getGraph(); + Graph graph = MODEL_GEOSPARQL.getGraph(); Resource targetSpatialObject = GEOMETRY_B; SpatialObjectGeometryLiteral expResult = new SpatialObjectGeometryLiteral(GEOMETRY_B.asNode(), LITERAL_B.asNode()); SpatialObjectGeometryLiteral result = SpatialObjectGeometryLiteral.retrieve(graph, targetSpatialObject.asNode()); @@ -102,7 +105,7 @@ public void testRetrieveGeometryLiteral_feature() { Resource targetSpatialObject = FEATURE_B; SpatialObjectGeometryLiteral expResult = new SpatialObjectGeometryLiteral(FEATURE_B.asNode(), LITERAL_B.asNode()); - SpatialObjectGeometryLiteral result = SpatialObjectGeometryLiteral.retrieve(MODEL.getGraph(), targetSpatialObject.asNode()); + SpatialObjectGeometryLiteral result = SpatialObjectGeometryLiteral.retrieve(MODEL_GEOSPARQL.getGraph(), targetSpatialObject.asNode()); assertEquals(expResult, result); } @@ -114,7 +117,7 @@ public void testRetrieveGeometryLiteral_missing_property() { Resource targetSpatialObject = ResourceFactory.createResource("http://example.org#GeometryE"); - SpatialObjectGeometryLiteral instance = SpatialObjectGeometryLiteral.retrieve(MODEL.getGraph(), targetSpatialObject.asNode()); + SpatialObjectGeometryLiteral instance = SpatialObjectGeometryLiteral.retrieve(MODEL_GEOSPARQL.getGraph(), targetSpatialObject.asNode()); boolean expResult = false; boolean result = instance.isValid(); @@ -129,7 +132,7 @@ public void testRetrieveGeometryLiteral_not_feature_geometry() { Resource targetSpatialObject = ResourceFactory.createResource("http://example.org#X"); - SpatialObjectGeometryLiteral instance = SpatialObjectGeometryLiteral.retrieve(MODEL.getGraph(), targetSpatialObject.asNode()); + SpatialObjectGeometryLiteral instance = SpatialObjectGeometryLiteral.retrieve(MODEL_GEOSPARQL.getGraph(), targetSpatialObject.asNode()); boolean expResult = false; boolean result = instance.isValid(); @@ -144,7 +147,7 @@ public void testRetrieveGeometryLiteral_feature_lat_lon() { Resource targetSpatialObject = GEO_FEATURE_Y; SpatialObjectGeometryLiteral expResult = new SpatialObjectGeometryLiteral(GEO_FEATURE_Y.asNode(), GEO_FEATURE_LITERAL.asNode()); - SpatialObjectGeometryLiteral result = SpatialObjectGeometryLiteral.retrieve(MODEL.getGraph(), targetSpatialObject.asNode()); + SpatialObjectGeometryLiteral result = SpatialObjectGeometryLiteral.retrieve(MODEL_WGS84.getGraph(), targetSpatialObject.asNode()); assertEquals(expResult, result); } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/AbstractSpatialIndexGraphLookpTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/AbstractSpatialIndexGraphLookpTest.java new file mode 100644 index 00000000000..5ec77e72ef3 --- /dev/null +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/AbstractSpatialIndexGraphLookpTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial; + +import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; +import org.apache.jena.query.ResultSet; +import org.apache.jena.query.ResultSetFormatter; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParserBuilder; +import org.apache.jena.sparql.algebra.Table; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.exec.QueryExec; +import org.junit.Assert; +import org.junit.Test; + +public abstract class AbstractSpatialIndexGraphLookpTest { + protected abstract SpatialIndex buildSpatialIndex(DatasetGraph dsg, String srsUri) throws SpatialIndexException; + + private static boolean enableDebugPrint = false; + + private static void debugPrint(Table table) { + if (enableDebugPrint) { + System.err.println(ResultSetFormatter.asText(ResultSet.adapt(table.toRowSet()))); + } + } + + // SpatialIndexUtils.buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + @Test + public void mustNotMatchDefaultGraph1() throws SpatialIndexException { + DatasetGraph dsg = RDFParserBuilder.create().fromString( """ + PREFIX eg: + PREFIX geo: + + eg:graph1 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (0.3 0.3)"^^geo:wktLiteral . + } + + eg:graph2 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (0.7 0.7)"^^geo:wktLiteral . + } + """).lang(Lang.TRIG).toDatasetGraph(); + + String queryStr = """ + PREFIX eg: + PREFIX spatial: + PREFIX geo: + + SELECT * { + VALUES ?search { + "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"^^geo:wktLiteral + } + LATERAL { + # GRAPH eg:graph1 { ?feature spatial:intersectBoxGeom(?search) . } + ?feature spatial:intersectBoxGeom(?search) . + } + } + """; + + buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + Table table = QueryExec.dataset(dsg).query(queryStr).table(); + debugPrint(table); + Assert.assertTrue(table.isEmpty()); + } + + @Test + public void mustNotMatchDefaultGraph2() throws SpatialIndexException { + DatasetGraph dsg = RDFParserBuilder.create().fromString( """ + PREFIX eg: + PREFIX geo: + + # Feature in default graph is outside of query polygon + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (-10 -10)"^^geo:wktLiteral . + + eg:graph1 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (0.3 0.3)"^^geo:wktLiteral . + } + + eg:graph2 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (0.7 0.7)"^^geo:wktLiteral . + } + """).lang(Lang.TRIG).toDatasetGraph(); + + String queryStr = """ + PREFIX eg: + PREFIX spatial: + PREFIX geo: + + SELECT * { + VALUES ?search { + "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"^^geo:wktLiteral + } + LATERAL { + ?feature spatial:intersectBoxGeom(?search) . + } + } + """; + + buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + Table table = QueryExec.dataset(dsg).query(queryStr).table(); + debugPrint(table); + Assert.assertTrue(table.isEmpty()); + } + + @Test + public void mustNotMatchNamedGraph() throws SpatialIndexException { + DatasetGraph dsg = RDFParserBuilder.create().fromString( """ + PREFIX eg: + PREFIX geo: + + eg:graph1 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (-10 -10)"^^geo:wktLiteral . + } + + eg:graph2 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (0.7 0.7)"^^geo:wktLiteral . + } + """).lang(Lang.TRIG).toDatasetGraph(); + + String queryStr = """ + PREFIX eg: + PREFIX spatial: + PREFIX geo: + + SELECT * { + VALUES ?search { + "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"^^geo:wktLiteral + } + LATERAL { + GRAPH eg:graph1 { ?feature spatial:intersectBoxGeom(?search) . } + } + } + """; + + buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + Table table = QueryExec.dataset(dsg).query(queryStr).table(); + debugPrint(table); + Assert.assertTrue(table.isEmpty()); + } + + @Test + public void mustMatchNamedGraph() throws SpatialIndexException { + DatasetGraph dsg = RDFParserBuilder.create().fromString( """ + PREFIX eg: + PREFIX geo: + + eg:graph1 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (-10 -10)"^^geo:wktLiteral . + } + + eg:graph2 { + eg:feature1 geo:hasGeometry eg:geometry1 . + eg:geometry1 geo:asWKT "POINT (0.7 0.7)"^^geo:wktLiteral . + } + """).lang(Lang.TRIG).toDatasetGraph(); + + String queryStr = """ + PREFIX eg: + PREFIX spatial: + PREFIX geo: + + SELECT * { + VALUES ?search { + "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"^^geo:wktLiteral + } + LATERAL { + GRAPH eg:graph2 { ?feature spatial:intersectBoxGeom(?search) . } + } + } + """; + + buildSpatialIndex(dsg, SRS_URI.DEFAULT_WKT_CRS84); + Table table = QueryExec.dataset(dsg).query(queryStr).table(); + debugPrint(table); + Assert.assertFalse(table.isEmpty()); + } +} diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SearchEnvelopeTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SearchEnvelopeTest.java index 371f18f1ad0..2d9a4f394c2 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SearchEnvelopeTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SearchEnvelopeTest.java @@ -17,15 +17,17 @@ */ package org.apache.jena.geosparql.spatial; -import java.util.Arrays; -import java.util.HashSet; +import static org.junit.Assert.assertEquals; + +import java.util.Collection; +import java.util.Set; + import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.implementation.datatype.WKTDatatype; import org.apache.jena.geosparql.implementation.vocabulary.Unit_URI; -import org.apache.jena.rdf.model.Resource; +import org.apache.jena.graph.Node; import org.junit.After; import org.junit.AfterClass; -import static org.junit.Assert.assertEquals; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -364,8 +366,8 @@ public void testCheck() { SearchEnvelope instance = SearchEnvelope.build(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO, radius, unitsURI); //Function Test - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.LONDON_FEATURE)); - HashSet result = instance.check(spatialIndex); + Set expResult = Set.of(SpatialIndexTestData.LONDON_FEATURE.asNode()); + Collection result = instance.check(spatialIndex); assertEquals(expResult, result); } @@ -384,8 +386,8 @@ public void testCheck_empty() { SearchEnvelope instance = SearchEnvelope.build(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO, radius, unitsURI); //Function Test - HashSet expResult = new HashSet<>(); - HashSet result = instance.check(spatialIndex); + Collection expResult = Set.of(); + Collection result = instance.check(spatialIndex); assertEquals(expResult, result); } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SpatialIndexTestData.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SpatialIndexTestData.java index b3ec8949988..060023a5480 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SpatialIndexTestData.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/SpatialIndexTestData.java @@ -17,17 +17,28 @@ */ package org.apache.jena.geosparql.spatial; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.implementation.SRSInfo; import org.apache.jena.geosparql.implementation.datatype.WKTDatatype; import org.apache.jena.geosparql.implementation.vocabulary.Geo; import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; +import org.apache.jena.geosparql.spatial.index.v2.STRtreePerGraph; +import org.apache.jena.geosparql.spatial.index.v2.STRtreeUtils; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexPerGraph; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; +import org.apache.jena.graph.Node; import org.apache.jena.query.Dataset; import org.apache.jena.query.DatasetFactory; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.ResourceFactory; +import org.locationtech.jts.index.strtree.STRtree; /** * @@ -60,21 +71,28 @@ public class SpatialIndexTestData { public static final SRSInfo WGS_84_SRS_INFO = new SRSInfo(SRS_URI.WGS84_CRS); public static final SRSInfo OSGB_SRS_INFO = new SRSInfo(SRS_URI.OSGB36_CRS); - private static SpatialIndex TEST_SPATIAL_INDEX = null; + private static SpatialIndexPerGraph TEST_SPATIAL_INDEX = null; private static Dataset TEST_DATASET = null; - public static final SpatialIndex createTestIndex() { + public static final List getTestItems() { + List items = List.of( + SpatialIndexItem.of(LONDON_GEOMETRY_WRAPPER.getEnvelope(), LONDON_FEATURE.asNode()), + SpatialIndexItem.of(NEW_YORK_GEOMETRY_WRAPPER.getEnvelope(), NEW_YORK_FEATURE.asNode()), + SpatialIndexItem.of(HONOLULU_GEOMETRY_WRAPPER.getEnvelope(), HONOLULU_FEATURE.asNode()), + SpatialIndexItem.of(PERTH_GEOMETRY_WRAPPER.getEnvelope(), PERTH_FEATURE.asNode()), + SpatialIndexItem.of(AUCKLAND_GEOMETRY_WRAPPER.getEnvelope(), AUCKLAND_FEATURE.asNode())); + return items; + } + + public static final SpatialIndexPerGraph createTestIndex() { if (TEST_SPATIAL_INDEX == null) { try { - SpatialIndex spatialIndex = new SpatialIndex(100, SRS_URI.WGS84_CRS); - spatialIndex.insertItem(LONDON_GEOMETRY_WRAPPER.getEnvelope(), LONDON_FEATURE); - spatialIndex.insertItem(NEW_YORK_GEOMETRY_WRAPPER.getEnvelope(), NEW_YORK_FEATURE); - spatialIndex.insertItem(HONOLULU_GEOMETRY_WRAPPER.getEnvelope(), HONOLULU_FEATURE); - spatialIndex.insertItem(PERTH_GEOMETRY_WRAPPER.getEnvelope(), PERTH_FEATURE); - spatialIndex.insertItem(AUCKLAND_GEOMETRY_WRAPPER.getEnvelope(), AUCKLAND_FEATURE); - - spatialIndex.build(); + // SpatialIndexPerGraph spatialIndex = new SpatialIndexPerGraph(100, SRS_URI.WGS84_CRS); + List items = getTestItems(); + STRtree tree = STRtreeUtils.buildSpatialIndexTree(items); + STRtreePerGraph index = new STRtreePerGraph(tree); + SpatialIndexPerGraph spatialIndex = new SpatialIndexPerGraph(index); TEST_SPATIAL_INDEX = spatialIndex; } catch (SpatialIndexException ex) { @@ -102,11 +120,15 @@ public static final Dataset createTestDataset() { dataset.setDefaultModel(model); SpatialIndex spatialIndex = createTestIndex(); - SpatialIndex.setSpatialIndex(dataset, spatialIndex); + SpatialIndexLib.setSpatialIndex(dataset, spatialIndex); TEST_DATASET = dataset; } return TEST_DATASET; } + public static Set asNodes(Collection resources) { + return resources.stream().map(Resource::asNode).collect(Collectors.toSet()); + } + } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/TestSpatialIndexGraphLookupV1.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/TestSpatialIndexGraphLookupV1.java new file mode 100644 index 00000000000..3bbd8756207 --- /dev/null +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/TestSpatialIndexGraphLookupV1.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial; + +import org.apache.jena.geosparql.spatial.index.v1.SpatialIndexAdapterV1; +import org.apache.jena.geosparql.spatial.index.v1.SpatialIndexV1; +import org.apache.jena.query.DatasetFactory; +import org.apache.jena.sparql.core.DatasetGraph; +import org.junit.Ignore; + +@SuppressWarnings("removal") +@Ignore /** These tests reveal issues with the first version of the spatial index which did not use per-graph indexes. */ +public class TestSpatialIndexGraphLookupV1 + extends AbstractSpatialIndexGraphLookpTest +{ + @Override + protected SpatialIndex buildSpatialIndex(DatasetGraph dsg, String srsUri) throws SpatialIndexException { + SpatialIndexV1 v1 = SpatialIndexV1.buildSpatialIndex(DatasetFactory.wrap(dsg), srsUri); + return new SpatialIndexAdapterV1(v1); + } +} diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/TestSpatialIndexGraphLookupV2.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/TestSpatialIndexGraphLookupV2.java new file mode 100644 index 00000000000..c23f6ff389b --- /dev/null +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/TestSpatialIndexGraphLookupV2.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial; + +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; +import org.apache.jena.sparql.core.DatasetGraph; + +public class TestSpatialIndexGraphLookupV2 + extends AbstractSpatialIndexGraphLookpTest +{ + @Override + protected SpatialIndex buildSpatialIndex(DatasetGraph dsg, String srsUri) throws SpatialIndexException { + return SpatialIndexLib.buildSpatialIndex(dsg, srsUri); + } +} diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/SimpleGraphSerializer.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/SimpleGraphSerializer.java new file mode 100644 index 00000000000..8235ad30203 --- /dev/null +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/SimpleGraphSerializer.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Triple; +import org.apache.jena.sparql.graph.GraphFactory; +import org.apache.jena.util.iterator.ExtendedIterator; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +/** Only used for testing Node_Graph serialization. */ +public class SimpleGraphSerializer + extends Serializer +{ + @Override + public void write(Kryo kryo, Output output, Graph graph) { + ExtendedIterator it = graph.find(); + try { + it.forEach(t -> kryo.writeObjectOrNull(output, t, Triple.class)); + } finally { + it.close(); + } + kryo.writeObjectOrNull(output, null, Triple.class); + } + + @Override + public Graph read(Kryo kryo, Input input, Class type) { + Graph result = GraphFactory.createDefaultGraph(); + for (;;) { + Triple t = kryo.readObjectOrNull(input, Triple.class); + if (t == null) { + break; + } + result.add(t); + } + return result; + } +} diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexTest.java new file mode 100644 index 00000000000..107d89a5ca0 --- /dev/null +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/SpatialIndexTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.jena.geosparql.implementation.GeometryWrapperFactory; +import org.apache.jena.geosparql.implementation.SRSInfo; +import org.apache.jena.geosparql.implementation.datatype.WKTDatatype; +import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; +import org.apache.jena.geosparql.spatial.SearchEnvelope; +import org.apache.jena.geosparql.spatial.SpatialIndex; +import org.apache.jena.geosparql.spatial.SpatialIndexException; +import org.apache.jena.geosparql.spatial.SpatialIndexItem; +import org.apache.jena.geosparql.spatial.SpatialIndexTestData; +import org.apache.jena.geosparql.spatial.index.compat.SpatialIndexIo; +import org.apache.jena.geosparql.spatial.index.v1.SpatialIndexV1; +import org.apache.jena.geosparql.spatial.index.v2.GeometryGenerator.GeometryType; +import org.apache.jena.graph.Node; +import org.apache.jena.query.DatasetFactory; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.junit.Assert; +import org.junit.Test; +import org.locationtech.jts.geom.Envelope; + +@SuppressWarnings("removal") +public class SpatialIndexTest { + + @Test + public void testLegacyLoading() throws IOException, SpatialIndexException { + Path file = Files.createTempFile("jena-", ".spatial.index"); + try { + List items = SpatialIndexTestData.getTestItems(); + + SpatialIndexV1.save(file.toFile(), items, SRS_URI.DEFAULT_WKT_CRS84); + + SpatialIndex index = SpatialIndexIo.load(file, true); + Envelope envelope = new Envelope(-90, 0, 0, 90); + Collection actual = index.query(envelope, null); + Set expected = Set.of(SpatialIndexTestData.LONDON_FEATURE.asNode(), SpatialIndexTestData.NEW_YORK_FEATURE.asNode()); + Assert.assertEquals(expected, actual); + } finally { + Files.delete(file); + } + } + + @Test + public void testSerdeSpatialIndex() throws IOException, SpatialIndexException { + // create spatial index + SpatialIndexPerGraph index1 = SpatialIndexTestData.createTestIndex(); + + // query index 1 + SRSInfo srsInfo1 = index1.getSrsInfo(); + SearchEnvelope searchEnvelope1 = SearchEnvelope.build(GeometryWrapperFactory.createPolygon(srsInfo1.getDomainEnvelope(), WKTDatatype.URI), srsInfo1); + Collection res1 = searchEnvelope1.check(index1); + + // save to tmp file + // File file = new File("/tmp/test-spatial.index"); //File.createTempFile( "jena", "spatial.index"); + Path file = Files.createTempFile("jena-", ".spatial-index"); + try { + SpatialIndexIoKryo.save(file, index1); + + // load from tmp file as new index 2 + SpatialIndex index2 = SpatialIndexIo.load(file); + + // query index 2 + SRSInfo srsInfo2 = index2.getSrsInfo(); + SearchEnvelope searchEnvelope2 = SearchEnvelope.build(GeometryWrapperFactory.createPolygon(srsInfo2.getDomainEnvelope(), WKTDatatype.URI), srsInfo2); + Collection res2 = searchEnvelope2.check(index2); + + assertEquals(srsInfo1, srsInfo2); + assertEquals(res1, res2); + } finally { + Files.deleteIfExists(file); + } + } + + /** + * Generate a dataset with geometries of all sorts, + * build the spatial index for it and + * attempt to save/load it. + */ + @Test + public void testSerdeSpatialIndex2() throws IOException, SpatialIndexException { + DatasetGraph dsg = DatasetGraphFactory.create(); + + Envelope envelope = new Envelope(-170, 170, -85, 85); + Map config = GeometryGenerator.createConfig(100); + long expectedItemCount = GeometryGenerator.generateGraph(dsg.getDefaultGraph(), envelope, config); + + Path file = Files.createTempFile("jena-", ".spatial-index"); + try { + // Query the built index (no IO) + SpatialIndexPerGraph indexA = (SpatialIndexPerGraph)SpatialIndexLib.buildSpatialIndex(dsg); + long itemCountA = indexA.query(envelope, null).size(); + Assert.assertEquals(expectedItemCount, itemCountA); + + // Save the index with the latest format and load from file. + SpatialIndexIoKryo.save(file, indexA); + SpatialIndex indexB = SpatialIndexIoKryo.load(file); + long itemCountB = indexB.query(envelope, null).size(); + Assert.assertEquals(expectedItemCount, itemCountB); + + // Save the index with legacy format and load from file. + // File must not exist or the index might/will attempt to load it. + Files.delete(file); + SpatialIndexV1.buildSpatialIndex(DatasetFactory.wrap(dsg), file.toFile()); + SpatialIndex indexC = SpatialIndexIo.load(file, true); + long itemCountC = indexC.query(envelope, null).size(); + Assert.assertEquals(expectedItemCount, itemCountC); + } finally { + Files.deleteIfExists(file); + } + } +} diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/TestNodeSerializer.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/TestNodeSerializer.java new file mode 100644 index 00000000000..69b546540ce --- /dev/null +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/index/v2/TestNodeSerializer.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.geosparql.spatial.index.v2; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.apache.jena.datatypes.BaseDatatype; +import org.apache.jena.datatypes.xsd.XSDDatatype; +import org.apache.jena.geosparql.kryo.NodeSerializer; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.graph.Node_Graph; +import org.apache.jena.graph.TextDirection; +import org.apache.jena.graph.Triple; +import org.apache.jena.graph.impl.GraphMatcher; +import org.apache.jena.sparql.core.Var; +import org.apache.jena.sparql.graph.GraphFactory; +import org.apache.jena.vocabulary.RDF; +import org.junit.Assert; +import org.junit.Test; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +public class TestNodeSerializer { + + private static final Node S = NodeFactory.createURI("http://www.example.org/s"); + private static final Node B = NodeFactory.createBlankNode("abcde"); + private static final Triple T = Triple.create(S, RDF.Nodes.type, B); + private static final Graph G = GraphFactory.createDefaultGraph(); + + private static final Kryo kryo = new Kryo(); + + static { + G.add(T); + NodeSerializer.register(kryo); + KryoRegistratorSpatialIndexV2.registerTripleSerializer(kryo); + registerGraphSerializer(kryo); + } + + @Test public void testUri() { assertRoundtrip(S); } + + @Test public void testBnode() { assertRoundtrip(NodeFactory.createBlankNode()); } + + @Test public void testLitStr() { assertRoundtrip(NodeFactory.createLiteralString("hello")); } + @Test public void testLitDtStd() { assertRoundtrip(NodeFactory.createLiteralDT("1", XSDDatatype.XSDinteger)); } + @Test public void testLitDtCustom() { assertRoundtrip(NodeFactory.createLiteralDT("1", new BaseDatatype("http://www.example.org/myDatatype"))); } + @Test public void testLitLang() { assertRoundtrip(NodeFactory.createLiteralLang("hello", "en")); } + + @Test public void testLitDirLangNone() { assertRoundtrip(NodeFactory.createLiteralDirLang("hello", "en", Node.noTextDirection)); } + @Test public void testLitDirLangLtr() { assertRoundtrip(NodeFactory.createLiteralDirLang("hello", "en", TextDirection.LTR)); } + @Test public void testLitDirLangRtl() { assertRoundtrip(NodeFactory.createLiteralDirLang("hello", "en", TextDirection.RTL)); } + + @Test public void testStar() { assertRoundtrip(NodeFactory.createTripleTerm(T)); } + + @Test public void testVarRdf() { assertRoundtrip(NodeFactory.createVariable("rdfVar")); } + @Test public void testVarSparql() { assertRoundtrip(Var.alloc("sparqlVar")); } + + @Test public void testAny() { assertRoundtrip(Node.ANY); } + + @Test public void testGraph() { + Node_Graph after = (Node_Graph)roundtrip(NodeFactory.createGraphNode(G)); + Assert.assertTrue(GraphMatcher.equals(G, after.getGraph())); + } + + // For completeness also test Node_Graph serialization + private static void registerGraphSerializer(Kryo kryo) { + Graph prototype = GraphFactory.createDefaultGraph(); + Class prototypeClass = prototype.getClass(); + Serializer graphSerializer = new SimpleGraphSerializer(); + kryo.register(Graph.class, graphSerializer); + kryo.register(prototypeClass, graphSerializer); + } + + public static Node roundtrip(Node expected) { + byte[] bytes; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + try (Output out = new Output(baos)) { + kryo.writeClassAndObject(out, expected); + out.flush(); + } + bytes = baos.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + Node result; + try (Input in = new Input(new ByteArrayInputStream(bytes))) { + result = (Node)kryo.readClassAndObject(in); + } + return result; + } + + public static void assertRoundtrip(Node expected) { + Node actual = roundtrip(expected); + Assert.assertEquals(expected, actual); + } +} diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunctionTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunctionTest.java index 724f4d2dacc..ec16c8000e4 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunctionTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/GenericSpatialPropertyFunctionTest.java @@ -17,15 +17,18 @@ */ package org.apache.jena.geosparql.spatial.property_functions; +import static org.junit.Assert.assertEquals; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; + import org.apache.jena.datatypes.xsd.XSDDatatype; import org.apache.jena.geosparql.configuration.GeoSPARQLConfig; import org.apache.jena.geosparql.implementation.vocabulary.SpatialExtension; -import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexException; import org.apache.jena.geosparql.spatial.SpatialIndexTestData; +import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexLib; import org.apache.jena.query.Dataset; import org.apache.jena.query.QueryExecution; import org.apache.jena.query.QueryExecutionFactory; @@ -37,7 +40,6 @@ import org.apache.jena.rdf.model.ResourceFactory; import org.junit.After; import org.junit.AfterClass; -import static org.junit.Assert.*; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -201,7 +203,7 @@ public void testExecEvaluated_Nearby_geo() throws SpatialIndexException { Resource geoFeature = ResourceFactory.createResource("http://example.org/GeoFeatureX"); model.add(geoFeature, SpatialExtension.GEO_LAT_PROP, ResourceFactory.createTypedLiteral("0.0", XSDDatatype.XSDfloat)); model.add(geoFeature, SpatialExtension.GEO_LON_PROP, ResourceFactory.createTypedLiteral("0.0", XSDDatatype.XSDfloat)); - Dataset dataset = SpatialIndex.wrapModel(model); + Dataset dataset = SpatialIndexLib.wrapModel(model); String query = "PREFIX spatial: \n" + "\n" diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastGeomPFTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastGeomPFTest.java index 91e6cb00e73..ef707118c1b 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastGeomPFTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastGeomPFTest.java @@ -17,16 +17,21 @@ */ package org.apache.jena.geosparql.spatial.property_functions.cardinal; +import static org.junit.Assert.assertEquals; + import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; +import java.util.Collection; import java.util.List; +import java.util.Set; + import org.apache.jena.geosparql.configuration.GeoSPARQLConfig; import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.spatial.CardinalDirection; import org.apache.jena.geosparql.spatial.SearchEnvelope; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexTestData; +import org.apache.jena.graph.Node; import org.apache.jena.query.Dataset; import org.apache.jena.query.QueryExecution; import org.apache.jena.query.QueryExecutionFactory; @@ -35,7 +40,6 @@ import org.apache.jena.rdf.model.Resource; import org.junit.After; import org.junit.AfterClass; -import static org.junit.Assert.*; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -91,8 +95,10 @@ public void testCheckSearchEnvelope_no_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.HONOLULU_GEOMETRY_WRAPPER; EastGeomPF instance = new EastGeomPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes( + List.of(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); + assertEquals(expResult, result); } @@ -108,8 +114,9 @@ public void testCheckSearchEnvelope_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.PERTH_GEOMETRY_WRAPPER; EastGeomPF instance = new EastGeomPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes( + List.of(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); assertEquals(expResult, result); } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastPFTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastPFTest.java index 38896523c06..32e04f0b2cf 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastPFTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/EastPFTest.java @@ -17,16 +17,15 @@ */ package org.apache.jena.geosparql.spatial.property_functions.cardinal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; +import java.util.*; + import org.apache.jena.geosparql.configuration.GeoSPARQLConfig; import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.spatial.CardinalDirection; import org.apache.jena.geosparql.spatial.SearchEnvelope; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexTestData; +import org.apache.jena.graph.Node; import org.apache.jena.query.Dataset; import org.apache.jena.query.QueryExecution; import org.apache.jena.query.QueryExecutionFactory; @@ -91,8 +90,8 @@ public void testCheckSearchEnvelope_no_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.HONOLULU_GEOMETRY_WRAPPER; EastPF instance = new EastPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes(List.of(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); assertEquals(expResult, result); } @@ -108,8 +107,8 @@ public void testCheckSearchEnvelope_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.PERTH_GEOMETRY_WRAPPER; EastPF instance = new EastPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes(List.of(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE, SpatialIndexTestData.NEW_YORK_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); assertEquals(expResult, result); } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestGeomPFTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestGeomPFTest.java index abda7afebf2..7ebe0276a8a 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestGeomPFTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestGeomPFTest.java @@ -17,16 +17,21 @@ */ package org.apache.jena.geosparql.spatial.property_functions.cardinal; +import static org.junit.Assert.assertEquals; + import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; +import java.util.Collection; import java.util.List; +import java.util.Set; + import org.apache.jena.geosparql.configuration.GeoSPARQLConfig; import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.spatial.CardinalDirection; import org.apache.jena.geosparql.spatial.SearchEnvelope; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexTestData; +import org.apache.jena.graph.Node; import org.apache.jena.query.Dataset; import org.apache.jena.query.QueryExecution; import org.apache.jena.query.QueryExecutionFactory; @@ -35,7 +40,6 @@ import org.apache.jena.rdf.model.Resource; import org.junit.After; import org.junit.AfterClass; -import static org.junit.Assert.*; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -91,8 +95,8 @@ public void testCheckSearchEnvelope_no_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.PERTH_GEOMETRY_WRAPPER; WestGeomPF instance = new WestGeomPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.PERTH_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes(List.of(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.PERTH_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); assertEquals(expResult, result); } @@ -108,8 +112,8 @@ public void testCheckSearchEnvelope_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.HONOLULU_GEOMETRY_WRAPPER; WestGeomPF instance = new WestGeomPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes(List.of(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); assertEquals(expResult, result); } diff --git a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestPFTest.java b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestPFTest.java index 4aea6a59698..ce70c2212e3 100644 --- a/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestPFTest.java +++ b/jena-geosparql/src/test/java/org/apache/jena/geosparql/spatial/property_functions/cardinal/WestPFTest.java @@ -17,16 +17,15 @@ */ package org.apache.jena.geosparql.spatial.property_functions.cardinal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; +import java.util.*; + import org.apache.jena.geosparql.configuration.GeoSPARQLConfig; import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.spatial.CardinalDirection; import org.apache.jena.geosparql.spatial.SearchEnvelope; import org.apache.jena.geosparql.spatial.SpatialIndex; import org.apache.jena.geosparql.spatial.SpatialIndexTestData; +import org.apache.jena.graph.Node; import org.apache.jena.query.Dataset; import org.apache.jena.query.QueryExecution; import org.apache.jena.query.QueryExecutionFactory; @@ -91,8 +90,8 @@ public void testCheckSearchEnvelope_no_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.PERTH_GEOMETRY_WRAPPER; WestPF instance = new WestPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.PERTH_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes(List.of(SpatialIndexTestData.LONDON_FEATURE, SpatialIndexTestData.PERTH_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); assertEquals(expResult, result); } @@ -108,8 +107,8 @@ public void testCheckSearchEnvelope_wrap() { GeometryWrapper geometryWrapper = SpatialIndexTestData.HONOLULU_GEOMETRY_WRAPPER; WestPF instance = new WestPF(); SearchEnvelope searchEnvelope = instance.buildSearchEnvelope(geometryWrapper, SpatialIndexTestData.WGS_84_SRS_INFO); //Needed to initialise the search. - HashSet expResult = new HashSet<>(Arrays.asList(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE)); - HashSet result = searchEnvelope.check(spatialIndex); + Set expResult = SpatialIndexTestData.asNodes(List.of(SpatialIndexTestData.AUCKLAND_FEATURE, SpatialIndexTestData.PERTH_FEATURE, SpatialIndexTestData.HONOLULU_FEATURE)); + Collection result = searchEnvelope.check(spatialIndex); assertEquals(expResult, result); } diff --git a/pom.xml b/pom.xml index cbc533a9134..ea9f24abcfc 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ https://gitbox.apache.org/repos/asf/jena.git HEAD - + 17 @@ -70,7 +70,7 @@ 12.0.21 6.1.0 - @@ -115,6 +115,7 @@ 1.37 + 4.0.3 2.3.1 4.0.2 1.82 @@ -134,7 +135,7 @@ 0.7.0 true @@ -145,7 +146,7 @@ JIRA https://issues.apache.org/jira/browse/JENA - + Jena Users Mailing List @@ -186,7 +187,7 @@ apache-jena @@ -325,7 +326,7 @@ pom import - + junit junit @@ -338,7 +339,7 @@ commons-codec ${ver.commons-codec} - + commons-io commons-io @@ -350,7 +351,7 @@ commons- ${ver.commons-io} - + org.apache.commons commons-compress @@ -376,7 +377,7 @@ protobuf-java ${ver.protobuf} - + org.apache.thrift libthrift @@ -391,7 +392,7 @@ * - @@ -447,24 +448,24 @@ commons-logging commons-logging - + @@ -541,7 +542,7 @@ @@ -629,7 +630,7 @@ ${ver.slf4j} - @@ -740,7 +741,7 @@ ${ver.mockito} test - + org.awaitility awaitility @@ -761,6 +762,12 @@ + + com.esotericsoftware + kryo + ${ver.kryo} + + com.beust jcommander @@ -772,7 +779,7 @@ jakarta.xml.bind-api ${ver.jakarta-xml-bind} - + javax.xml.bind @@ -838,6 +845,23 @@ + + + + org.seleniumhq.selenium + selenium-java + 4.18.1 + test + + + + io.github.bonigarcia + webdrivermanager + 5.8.0 + test + + + @@ -889,7 +913,7 @@ org.apache.rat apache-rat-plugin ${ver.plugin.rat} - @@ -1013,9 +1037,9 @@