Misc enhancements to 5.x

.gitignore

@@ -0,0 +1,3 @@
+build-lib
+build
+dist

README.md

@@ -1,3 +1,23 @@
+# Notes about this fork
+
+This project is a fork of https://github.com/lucidworks/query-autofiltering-component, but includes the following changes to the 5.x code base:
+
+* Resolved issues with Ivy dependencies.
+* Upgraded component to work with Solr 5.3.1 and above.
+* Added support for using a field whitelist. 
+
+The whilelist field definition feature was implemented to solve a gap regarding dynamic fields. Although asking the Luke admin handler would have been another implementation option, it seems that a generic whitelist is more powerful; albeit, a bit more verbose in the configuration. Simply define the following:
+
+<pre>
+ &lt;searchComponent name="autofilter" class="org.apache.solr.handler.component.QueryAutoFilteringComponent" >
+   &lt;arr name="whitelistFields">
+   &lt;str>field1&lt;/str>
+   &lt;str>field2&lt;/str>
+   &lt;str>fieldN&lt;/str>
+   &lt;/arr>
+ &lt;/searchComponent>
+</pre>
+
 # query-autofiltering-component
 A Query Autofiltering SearchComponent for Solr that can translate free-text queries into structured queries using index metadata.
 

solr5.x/build.xml

@@ -20,8 +20,8 @@
   <property name="project.name" value="${ant.project.name}" />
   <property name="version.number" value="1.0.1"/>
 
-  <property name="lucene.version" value="5.1.0"/>
-  <property name="solr.version" value="5.1.0"/>
+  <property name="lucene.version" value="5.3.1"/>
+  <property name="solr.version" value="5.3.1"/>
 
   <!-- change this to include ivy download dest -->
   <target name="set-classpaths" depends="ivy-retrieve" >

solr5.x/ivy/ivy-settings.xml

@@ -1,7 +1,11 @@
-
 <ivysettings>
-    <settings defaultResolver="central"/>
+    <settings defaultResolver="resolver-chain"/>
     <resolvers>
         <ibiblio name="central" m2compatible="true"/>
+        <ibiblio name="restlet" m2compatible="true" root="http://maven.restlet.com/"/>
+        <chain name="resolver-chain">
+            <resolver ref="central"/>
+            <resolver ref="restlet"/>
+        </chain>
     </resolvers>
 </ivysettings>

solr5.x/src/main/java/org/apache/solr/handler/component/QueryAutoFilteringComponent.java

@@ -98,6 +98,7 @@ public class QueryAutoFilteringComponent extends QueryComponent implements SolrC
 
   private String termsHandler = "/terms";
 
+  private HashSet<String> whitelistFields;;
   private HashSet<String> excludeFields;
   private HashSet<String> stopwords;
 
@@ -115,6 +116,15 @@ public class QueryAutoFilteringComponent extends QueryComponent implements SolrC
 
   @Override
   public void init( NamedList initArgs ) {
+
+    List<String> whitelistFields = (List<String>) initArgs.get("whitelistFields");
+    if (whitelistFields != null) {
+      this.whitelistFields = new HashSet<String>( );
+      for (String field : whitelistFields ) {
+          this.whitelistFields.add( field );
+      }
+    }
+
     List<String> excludeFields = (List<String>) initArgs.get("excludeFields");
     if (excludeFields != null) {
       this.excludeFields = new HashSet<String>( );
@@ -469,16 +479,15 @@ else if (qbuilder.length() > 0 && fieldMap.size() > 0) {
         }
       }
       else { // boostFactor is NOT null
-        // use the original query add fielded boost clauses
+        // use the bq field to add fielded boost clauses
         StringBuilder bbuilder = new StringBuilder( );
         String boostSuffix = "^" + boostFactor.toString( );
-        bbuilder.append( getPhrase( queryTokens, 0, queryTokens.size() - 1, " " ) );
         for (String fieldName : fieldMap.keySet( ) ) {
           bbuilder.append( " " );
           bbuilder.append( getFilterQuery( rb, fieldName, fieldMap.get( fieldName ), fieldPositionMap.get( fieldName ), queryTokens, boostSuffix ) );
         }
-        Log.info( "setting q = '" + bbuilder.toString()  + "'" );
-        modParams.set( "q", bbuilder.toString( ) );
+        Log.info( "adding bq = '" + bbuilder.toString()  + "'" );
+        modParams.add( "bq", bbuilder.toString( ).trim() );
       }
       return true;
     }
@@ -729,23 +738,35 @@ private void buildFieldMap( ResponseBuilder rb ) throws IOException {
 
   // TODO: Filter this by the configuration fields ...
   private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
-    IndexSchema schema = searcher.getSchema();
+
     ArrayList<String> strFields = new ArrayList<String>( );
-
-    Collection<String> fieldNames = searcher.getFieldNames();
-    Iterator<String> fnIt = fieldNames.iterator();
-    while ( fnIt.hasNext() ) {
-      String fieldName = fnIt.next( );
-      if (excludeFields == null || !excludeFields.contains( fieldName )) {
-        SchemaField field = schema.getField(fieldName);
-        if (field.stored() && field.getType() instanceof StrField ) {
-          strFields.add( fieldName );
+
+    if ( hasWhitelist() ) {
+      Log.info("Using whitelist fields instead of schema.");
+      for ( String fieldName: whitelistFields ) {
+        strFields.add( fieldName );
+      }
+    } else {
+      IndexSchema schema = searcher.getSchema();
+      Collection<String> fieldNames = searcher.getFieldNames();
+      Iterator<String> fnIt = fieldNames.iterator();
+      while ( fnIt.hasNext() ) {
+        String fieldName = fnIt.next( );
+        if (excludeFields == null || !excludeFields.contains( fieldName )) {
+          SchemaField field = schema.getField(fieldName);
+          if (field.stored() && field.getType() instanceof StrField ) {
+            strFields.add( fieldName );
+          }
         }
       }
     }
-      
+
     return strFields;
   }
+
+  private boolean hasWhitelist() {
+    return this.whitelistFields != null && this.whitelistFields.size() > 0;
+  }
 
   private void addTerm( CharsRef fieldChars, String fieldValue, SynonymMap.Builder fieldBuilder, SynonymMap.Builder termBuilder ) throws IOException {
 
@@ -804,11 +825,13 @@ private void addDistributedTerms( ResponseBuilder rb, SynonymMap.Builder fieldBu
 
     ShardHandlerFactory shardHandlerFactory = container.getShardHandlerFactory( );
     ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
-    shardHandler.checkDistributed( rb );
-
-    Log.debug( "Is Distributed = " + rb.isDistrib );
+
+    final SolrParams distribParams = rb.req.getParams(); 
+    final boolean isDistrib = distribParams.get(ShardParams.SHARDS) != null;
+    Log.debug( "Is Distributed = " + isDistrib );
 
-    if( rb.isDistrib ) {
+    if( isDistrib ) {
+      shardHandler.prepDistributed( rb );
       // create a ShardRequest that contains a Terms Request.
       // don't send to this shard???
       ShardRequest sreq = new ShardRequest();

solr5.x/src/test/resources/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml

@@ -31,7 +31,7 @@ A solrconfig.xml snippet containing indexConfig settings for randomized testing.
   <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
 
   <maxBufferedDocs>${solr.tests.maxBufferedDocs}</maxBufferedDocs>
-  <maxIndexingThreads>${solr.tests.maxIndexingThreads}</maxIndexingThreads>
+  <maxIndexingThreads>${solr.tests.maxIndexingThreads:8}</maxIndexingThreads>
   <ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
 
   <mergeScheduler class="${solr.tests.mergeScheduler}" />

solr6.x/build.xml

@@ -0,0 +1,127 @@
+<project name="query-autofiltering-component" default="dist" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant" >
+  <description>Builds Query Autofiltering Component</description>
+
+  <!-- set global properties for this build -->
+  <property name="build.dir" location="build"/>
+  <property name="dist.dir"  location="dist"/>
+  <property name="build.lib.dir" value="${basedir}/build-lib"/>
+
+  <property name="src.dir" location="src/main/java"/>
+  <property name="classes.dir" location="${build.dir}/java" />
+  <property name="test.dir" location="src/test" />
+  <property name="test.classes.dir" location="${build.dir}/test" />
+  <property name="test.resources" location="src/test/resources" />
+
+  <property name="ivy.dir" location="${basedir}/ivy"/>
+  <property name="ivy.dep.file" location="ivy.xml" />
+  <property name="ivy.version" value="2.3.0"/>
+  <property name="ivy.lib.dir"  location="${build.lib.dir}/ivy/lib" />
+
+  <property name="project.name" value="${ant.project.name}" />
+  <property name="version.number" value="1.0.1"/>
+
+  <property name="lucene.version" value="6.1.0"/>
+  <property name="solr.version" value="6.1.0"/>
+
+  <!-- change this to include ivy download dest -->
+  <target name="set-classpaths" depends="ivy-retrieve" >
+    <path id="compile-classpath" >
+      <fileset dir="${ivy.lib.dir}/compile" includes="*.jar"/> 
+    </path>
+
+    <path id="test-classpath">
+      <fileset dir="${ivy.lib.dir}/test" includes="*.jar"/>
+    </path>
+  </target>
+
+  <target name="init">
+    <!-- Create the build directory structure used by compile -->
+    <mkdir dir="${build.dir}"/>
+    <mkdir dir="${classes.dir}"/>
+    <mkdir dir="${test.classes.dir}"/>
+  </target>
+
+  <target name="compile" depends="init,ivy-retrieve,set-classpaths" description="compile the source " >
+    <!-- Compile the java code from ${src} into ${build} -->
+    <javac srcdir="${src.dir}" destdir="${classes.dir}" includeantruntime="false" >
+      <classpath>
+        <fileset dir="${ivy.lib.dir}" includes="compile/*.jar" />
+      </classpath>
+    </javac>
+  </target>
+
+  <target  name="compile-test" depends="compile" >
+    <javac srcdir="${test.dir}" destdir="${test.classes.dir}" includeantruntime="false" >
+      <classpath>
+        <fileset dir="${ivy.lib.dir}" includes="test/*.jar" />
+        <pathelement location="${classes.dir}" />
+      </classpath>
+    </javac>
+  </target>
+
+  <target name="dist" depends="compile,test" description="generate the distribution" >
+    <!-- Create the distribution directory -->
+    <mkdir dir="${dist.dir}"/>
+    <jar jarfile="${dist.dir}/${project.name}-${version.number}.jar" basedir="${classes.dir}"/>
+    <!-- Add JavaDocs -->
+
+  </target>
+
+  <target name="clean" description="clean up" >
+    <!-- Delete the ${build.dir} and ${dist.dir} directory trees -->
+    <delete dir="${build.dir}"/>
+    <delete dir="${dist.dir}"/>
+    <delete dir="${ivy.lib.dir}"/>
+  </target>
+
+  <target name="test" depends="compile,compile-test">
+    <echo message=""/>
+    <junit printsummary="yes" fork="yes" haltonfailure="yes">
+      <formatter usefile="false" type="plain"/>
+      <classpath>
+        <path refid="test-classpath"/>
+        <pathelement location="${classes.dir}"/>
+        <pathelement location="${test.classes.dir}" />
+        <pathelement location="${test.resources}" />
+        <pathelement location="${ivy.lib.dir}/junit-4.11.jar"/>
+      </classpath> 
+      <assertions>
+        <enable/>
+      </assertions>
+      <test name="org.apache.solr.handler.component.QueryAutoFilteringComponentTest"/>
+      <!-- test name="org.apache.solr.handler.component.DistributedQueryAutoFilteringTest" / -->
+    </junit>
+  </target>
+
+  <!-- Ivy -->
+  <condition property="ivy.jar.exists">
+    <or>
+      <available file="${build.lib.dir}/ivy-${ivy.version}.jar"/>
+      <isset property="offline"/>
+    </or>
+  </condition>
+
+  <target name="ivy-download" unless="ivy.jar.exists">
+    <mkdir dir="${build.lib.dir}"/>
+    <get src="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"
+         dest="${build.lib.dir}/ivy-${ivy.version}.jar"
+         usetimestamp="true"/>
+  </target>
+
+  <target name="ivy-init" depends="ivy-download" unless="skip.ivy" description="initialize ivy">
+    <path id="ivy.lib.path">
+      <fileset dir="${build.lib.dir}" includes="ivy-${ivy.version}.jar"/>
+    </path>
+    <taskdef resource="org/apache/ivy/ant/antlib.xml" uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path"/>
+    <ivy:settings file="${ivy.dir}/ivy-settings.xml"/>
+  </target>
+
+  <target name="ivy-resolve" depends="ivy-init" >
+    <ivy:resolve />
+  </target>
+
+  <target name="ivy-retrieve" depends="ivy-resolve" >
+    <ivy:retrieve pattern="${ivy.lib.dir}/[conf]/[artifact]-[revision].[ext]" sync="true" />
+  </target>
+
+</project>

solr6.x/ivy.xml

@@ -0,0 +1,21 @@
+<ivy-module version="2.0">
+  <info organisation="com.lucidworks.demo" module="autophrase-tokenfilter"/>
+
+  <configurations defaultconfmapping="default">
+    <conf name="default" extends="compile"/>
+    <conf name="compile"/>
+    <conf name="test" extends="compile" visibility="private"/>
+  </configurations>
+
+  <dependencies>
+    <dependency org="org.apache.lucene" name="lucene-analyzers-common" rev="${lucene.version}" conf="compile->default"/>
+    <dependency org="org.apache.lucene" name="lucene-core" rev="${lucene.version}" conf="compile->default"/>
+    <dependency org="org.apache.lucene" name="lucene-codecs" rev="${lucene.version}" conf="compile->default"/>
+    <dependency org="org.apache.solr" name="solr-core" rev="${lucene.version}" conf="compile->default" />
+    <dependency org="org.apache.solr" name="solr-solrj" rev="${lucene.version}" conf="compile->default" />
+    <dependency org="org.apache.solr" name="solr-test-framework" rev="${lucene.version}" conf="test->default" />
+    <dependency org="commons-logging" name="commons-logging" rev="1.2" conf="test->default" />
+    <dependency org="junit" name="junit" rev="4.11" conf="test->default" />
+    <dependency org="org.slf4j" name="slf4j-api" rev="1.7.6" conf="compile->default"/>
+  </dependencies>
+</ivy-module>

solr6.x/ivy/ivy-settings.xml

@@ -0,0 +1,11 @@
+<ivysettings>
+    <settings defaultResolver="resolver-chain"/>
+    <resolvers>
+        <ibiblio name="central" m2compatible="true"/>
+        <ibiblio name="restlet" m2compatible="true" root="http://maven.restlet.com/"/>
+        <chain name="resolver-chain">
+            <resolver ref="central"/>
+            <resolver ref="restlet"/>
+        </chain>
+    </resolvers>
+</ivysettings>