Improve citation cache memory efficiency

montysolr/build.gradle.kts

@@ -1,6 +1,7 @@
 plugins {
 	java
 	antlr
+    kotlin("jvm") version "1.9.22"
 }
 
 repositories {
@@ -23,10 +24,16 @@ dependencies {
 	implementation("com.anyascii:anyascii:0.3.2")
 	implementation("org.python:jython-standalone:2.7.3")
 
+	implementation("me.lemire.integercompression:JavaFastPFOR:0.1.12")
+	implementation("it.unimi.dsi:fastutil-core:8.5.12")
+
 	testImplementation("junit:junit:4.13.2")
 	testImplementation("org.antlr:stringtemplate:3.2.1")
 	testImplementation("org.apache.solr:solr-test-framework:7.7.3")
 	testImplementation("org.apache.lucene:lucene-test-framework:7.7.3")
+
+	testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0-RC2")
+    testImplementation(kotlin("stdlib-jdk8"))
 }
 
 java {

montysolr/src/main/java/org/apache/solr/handler/batch/BatchProviderDumpCitationCache.java

@@ -7,6 +7,8 @@
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.search.CitationCache;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.RefCounted;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -50,39 +52,52 @@ public void run(SolrQueryRequest req, BatchHandlerRequestQueue queue) throws Exc
 
 
         if (!returnDocids) {
-            SortedDocValues uniqueValueCache = req.getSearcher().getSlowAtomicReader().getSortedDocValues(uniqueField);
-            int paperid = 0;
-            while (it.hasNext()) {
-                int[][] data = it.next();
-                int[] references = data[0];
-                //TODO:rca - have a feeling this has become too convoluted
-                // and there must be a better way to un-invert; especially
-                // with docvalues
-                if (references != null && references.length > 0) {
-                    if (uniqueValueCache.advanceExact(paperid)) {
-                        ret = uniqueValueCache.binaryValue();
-                        out.write(ret.utf8ToString());
-                        out.write("\t");
-                        first = true;
-                        for (int luceneDocId : references) {
-                            if (luceneDocId == -1)
-                                continue;
-
-                            uniqueValueCache.advanceExact(luceneDocId);
+            RefCounted<SolrIndexSearcher> searcherRef = req.getCore().getRealtimeSearcher();
+            try {
+                SortedDocValues uniqueValueCache = searcherRef.get()
+                        .getSlowAtomicReader().getSortedDocValues(uniqueField);
+
+                int paperid = 0;
+                while (it.hasNext()) {
+                    int[][] data = it.next();
+                    int[] references = data[0];
+                    //TODO:rca - have a feeling this has become too convoluted
+                    // and there must be a better way to un-invert; especially
+                    // with docvalues
+                    if (references != null && references.length > 0) {
+                        if (uniqueValueCache.advanceExact(paperid)) {
                             ret = uniqueValueCache.binaryValue();
-
-                            if (ret.length > 0) {
-                                if (!first) {
-                                    out.write("\t");
+                            out.write(ret.utf8ToString());
+                            out.write("\t");
+                            first = true;
+
+                            // It's not possible to reset the SortedDocValues iterator, so create a new one and
+                            // seek to the appropriate point for each referenced document.
+                            SortedDocValues referenceValueCache = searcherRef.get()
+                                    .getSlowAtomicReader().getSortedDocValues(uniqueField);
+                            for (int luceneDocId : references) {
+                                if (luceneDocId == -1)
+                                    continue;
+
+                                if (referenceValueCache.advanceExact(luceneDocId)) {
+                                    ret = referenceValueCache.binaryValue();
+
+                                    if (ret.length > 0) {
+                                        if (!first) {
+                                            out.write("\t");
+                                        }
+                                        out.write(ret.utf8ToString());
+                                        first = false;
+                                    }
                                 }
-                                out.write(ret.utf8ToString());
-                                first = false;
                             }
+                            out.write("\n");
                         }
-                        out.write("\n");
                     }
+                    paperid++;
                 }
-                paperid++;
+            } finally {
+                searcherRef.decref();
             }
         } else {
             int paperid = 0;