Paginate using Lucene Doc ID

core/src/main/scala/ai/lum/odinson/ExtractorEngine.scala

@@ -136,20 +136,7 @@ class ExtractorEngine private (
     * @return
     */
   def query(odinsonQuery: OdinsonQuery, n: Int, disableMatchSelector: Boolean): OdinResults = {
-    query(odinsonQuery, n, null, disableMatchSelector)
-  }
-
-  /** Executes an OdinsonQuery and returns an OdinResult
-    * with the next `n` matched lucene documents and their corresponding matches,
-    * starting after the last lucene document in the OdinResults `after`.
-    *
-    * @param odinsonQuery
-    * @param n     number of desired lucene documents
-    * @param after an OdinResults with a set of lucene documents
-    * @return
-    */
-  def query(odinsonQuery: OdinsonQuery, n: Int, after: OdinResults): OdinResults = {
-    query(odinsonQuery, n, after.scoreDocs.last)
+    query(odinsonQuery, n, -1, disableMatchSelector)
   }
 
   /** Executes an OdinsonQuery and returns an OdinResult
@@ -158,10 +145,10 @@ class ExtractorEngine private (
     *
     * @param odinsonQuery
     * @param n     number of desired lucene documents
-    * @param after the last lucene document to ignore
+    * @param after the ID of the last lucene document to ignore
     * @return
     */
-  def query(odinsonQuery: OdinsonQuery, n: Int, after: OdinsonScoreDoc): OdinResults = {
+  def query(odinsonQuery: OdinsonQuery, n: Int, after: Int): OdinResults = {
     query(odinsonQuery, n, after, false)
   }
 
@@ -185,7 +172,7 @@ class ExtractorEngine private (
   def query(
     odinsonQuery: OdinsonQuery,
     n: Int,
-    after: OdinsonScoreDoc,
+    after: Int,
     disableMatchSelector: Boolean
   ): OdinResults = {
     val odinResults =
@@ -203,16 +190,16 @@ class ExtractorEngine private (
   }
 
   private def odinSearch(
-    after: OdinsonScoreDoc,
+    after: Int,
     query: OdinsonQuery,
     numHits: Int,
     disableMatchSelector: Boolean
   ): OdinResults = {
 
     val limit = math.max(1, index.maxDoc())
     require(
-      after == null || after.doc < limit,
-      s"after.doc exceeds the number of documents in the reader: after.doc=${after.doc} limit=${limit}"
+      after < limit,
+      s"after exceeds the number of documents in the reader: after=${after} limit=${limit}"
     )
     val cappedNumHits = math.min(numHits, limit)
     index.search(after, query, cappedNumHits, disableMatchSelector)
@@ -350,7 +337,7 @@ class ExtractorEngine private (
     disableMatchSelector: Boolean,
     mruIdGetter: MostRecentlyUsed[Int, LazyIdGetter]
   ): Iterator[Mention] = {
-    val odinResults = query(extractor.query, numSentences, null, disableMatchSelector)
+    val odinResults = query(extractor.query, numSentences, -1, disableMatchSelector)
     new MentionsIterator(
       extractor.label,
       Some(extractor.name),

core/src/main/scala/ai/lum/odinson/lucene/index/IncrementalOdinsonIndex.scala

@@ -265,13 +265,13 @@ class IncrementalOdinsonIndex(
   }
 
   override def search(
-    scoreDoc: OdinsonScoreDoc,
+    luceneDocId: Int,
     query: OdinsonQuery,
     cappedHits: Int,
     disableMatchSelector: Boolean
   ): OdinResults = {
     val manager =
-      new OdinsonCollectorManager(scoreDoc, cappedHits, computeTotalHits, disableMatchSelector)
+      new OdinsonCollectorManager(luceneDocId, cappedHits, computeTotalHits, disableMatchSelector)
     this.search(query, manager)
   }
 

core/src/main/scala/ai/lum/odinson/lucene/index/OdinsonCollectorManager.scala

@@ -8,7 +8,7 @@ import java.util.Collection
 import scala.collection.JavaConverters._
 
 class OdinsonCollectorManager(
-  after: OdinsonScoreDoc,
+  after: Int,
   cappedNumHits: Int,
   computeTotalHits: Boolean,
   disableMatchSelector: Boolean

core/src/main/scala/ai/lum/odinson/lucene/index/OdinsonIndex.scala

@@ -116,7 +116,7 @@ trait OdinsonIndex {
   ): ResultType
 
   def search(
-    scoreDoc: OdinsonScoreDoc,
+    luceneDocId: Int,
     query: OdinsonQuery,
     cappedHits: Int,
     disableMatchSelector: Boolean

core/src/main/scala/ai/lum/odinson/lucene/search/OdinsonCollector.scala

@@ -23,20 +23,6 @@ class OdinsonCollector(
     this(numHits, -1, computeTotalHits, disableMatchSelector)
   }
 
-  def this(
-    numHits: Int,
-    afterDoc: OdinsonScoreDoc,
-    computeTotalHits: Boolean,
-    disableMatchSelector: Boolean
-  ) = {
-    this(
-      numHits,
-      if (afterDoc == null) -1 else afterDoc.doc,
-      computeTotalHits,
-      disableMatchSelector
-    )
-  }
-
   private var totalHits: Int = 0
   private var collectedHits: Int = 0
 

core/src/main/scala/ai/lum/odinson/lucene/search/OdinsonIndexSearcher.scala

@@ -34,15 +34,15 @@ class OdinsonIndexSearcher(
   }
 
   def odinSearch(query: OdinsonQuery, n: Int): OdinResults = {
-    odinSearch(null, query, n)
+    odinSearch(-1, query, n)
   }
 
-  def odinSearch(after: OdinsonScoreDoc, query: OdinsonQuery, numHits: Int): OdinResults = {
+  def odinSearch(after: Int, query: OdinsonQuery, numHits: Int): OdinResults = {
     odinSearch(after, query, numHits, false)
   }
 
   class StandardCollectorManager(
-    after: OdinsonScoreDoc,
+    after: Int,
     cappedNumHits: Int,
     disableMatchSelector: Boolean
   ) extends CollectorManager[OdinsonCollector, OdinResults] {
@@ -60,15 +60,15 @@ class OdinsonIndexSearcher(
   }
 
   def odinSearch(
-    after: OdinsonScoreDoc,
+    after: Int,
     query: OdinsonQuery,
     numHits: Int,
     disableMatchSelector: Boolean
   ): OdinResults = {
     val limit = math.max(1, readerContext.reader().maxDoc())
     require(
-      after == null || after.doc < limit,
-      s"after.doc exceeds the number of documents in the reader: after.doc=${after.doc} limit=${limit}"
+      after < limit,
+      s"after exceeds the number of documents in the reader: after=${after} limit=${limit}"
     )
     val cappedNumHits = math.min(numHits, limit)
     val manager = new StandardCollectorManager(after, cappedNumHits, disableMatchSelector)

core/src/test/scala/ai/lum/odinson/events/TestEvents.scala

@@ -1,7 +1,7 @@
 package ai.lum.odinson.events
 
 import ai.lum.odinson.lucene.OdinResults
-import ai.lum.odinson.lucene.search.{ OdinsonQuery, OdinsonScoreDoc }
+import ai.lum.odinson.lucene.search.OdinsonQuery
 import ai.lum.odinson.test.utils.OdinsonTest
 import ai.lum.odinson.utils.exceptions.OdinsonException
 import ai.lum.odinson.{ EventMatch, MentionsIterator }
@@ -197,7 +197,7 @@ class TestEvents extends OdinsonTest {
       labelOpt: Option[String] = None,
       nameOpt: Option[String] = None,
       n: Int,
-      after: OdinsonScoreDoc,
+      after: Int,
       disableMatchSelector: Boolean
     ): OdinResults = {
       val odinResults = ee.query(odinsonQuery, n, after, disableMatchSelector)
@@ -214,14 +214,14 @@ class TestEvents extends OdinsonTest {
       labelOpt = Some("NP"),
       nameOpt = None,
       1,
-      after = null,
+      after = -1,
       disableMatchSelector = false
     )
     results1.totalHits should equal(1)
     results1.scoreDocs.head.matches should have size 2
 
     // This query only needs to read from the state.
-    val results2 = ee.query(q2, 1, after = null, disableMatchSelector = false)
+    val results2 = ee.query(q2, 1, after = -1, disableMatchSelector = false)
     results2.totalHits should equal(1)
     results2.scoreDocs.head.matches should have size 1
 

extra/src/main/scala/ai/lum/odinson/extra/Shell.scala

@@ -78,7 +78,7 @@ object Shell extends App {
   val matchTextToParse = """^:mkDoc\s+(.*)""".r
 
   var query: String = null
-  var after: OdinsonScoreDoc = null
+  var after: Int = -1
   var shownHits: Int = 0
   var totalHits: Int = 0
 
@@ -171,21 +171,26 @@ object Shell extends App {
     println(json)
   }
 
+  def getLastDocId(scoreDocs: Array[OdinsonScoreDoc]): Int = scoreDocs.lastOption match {
+    case Some(sd) => sd.doc
+    case _        => -1
+  }
+
   /** searches for pattern and prints the first n matches */
   def search(n: Int): Unit = {
     val start = System.currentTimeMillis()
     val q = extractorEngine.compiler.mkQuery(query)
     val results = extractorEngine.query(q, n)
     val duration = (System.currentTimeMillis() - start) / 1000f
-    after = results.scoreDocs.lastOption.getOrElse(null)
+    after = getLastDocId(results.scoreDocs)
     totalHits = results.totalHits
     shownHits = math.min(n, totalHits)
     printResultsPage(results, 1, totalHits, duration)
   }
 
   /** prints the next n matches */
   def printMore(n: Int): Unit = {
-    if (after == null) {
+    if (after == -1) {
       println("there is no active query")
       return
     }
@@ -197,8 +202,8 @@ object Shell extends App {
     val q = extractorEngine.compiler.mkQuery(query)
     val results = extractorEngine.query(q, n, after)
     val duration = (System.currentTimeMillis() - start) / 1000f
-    after = results.scoreDocs.lastOption.getOrElse(null)
-    if (after == null) {
+    after = getLastDocId(results.scoreDocs)
+    if (after == -1) {
       println("no more results")
       return
     }

project/build.properties

@@ -1 +1 @@
-sbt.version=1.6.0-RC1
+sbt.version=1.6.1