Merge pull request #300 from weaviate/feat/bm25-minimumShouldMatch

Extend BM25 and Hybrid search with minimum_should_match semantics.

Author: bevzzz

src/collections/query/index.ts

@@ -247,6 +247,7 @@ export {
   BaseHybridOptions,
   BaseNearOptions,
   BaseNearTextOptions,
+  Bm25OperatorOptions,
   Bm25Options,
   FetchObjectByIdOptions,
   FetchObjectsOptions,
@@ -266,3 +267,5 @@ export {
   QueryReturn,
   SearchOptions,
 } from './types.js';
+
+export { Bm25Operator } from './utils.js';

src/collections/query/integration.test.ts

@@ -1,10 +1,12 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 /* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */
+import { requireAtLeast } from '../../../test/version.js';
 import { WeaviateUnsupportedFeatureError } from '../../errors.js';
 import weaviate, { WeaviateClient } from '../../index.js';
 import { Collection } from '../collection/index.js';
 import { CrossReference, Reference } from '../references/index.js';
 import { GroupByOptions } from '../types/index.js';
+import { Bm25Operator } from './utils.js';
 
 describe('Testing of the collection.query methods with a simple collection', () => {
   let client: WeaviateClient;
@@ -132,6 +134,32 @@ describe('Testing of the collection.query methods with a simple collection', ()
     expect(ret.objects[0].uuid).toEqual(id);
   });
 
+  requireAtLeast(
+    1,
+    31,
+    0
+  )('bm25 search operator (minimum_should_match)', () => {
+    it('should query with bm25 + operator', async () => {
+      const ret = await collection.query.bm25('carrot', {
+        limit: 1,
+        operator: Bm25Operator.or({ minimumMatch: 1 }),
+      });
+      expect(ret.objects.length).toEqual(1);
+      expect(ret.objects[0].properties.testProp).toEqual('carrot');
+      expect(ret.objects[0].uuid).toEqual(id);
+    });
+
+    it('should query with hybrid + bm25Operator', async () => {
+      const ret = await collection.query.hybrid('carrot', {
+        limit: 1,
+        bm25Operator: Bm25Operator.and(),
+      });
+      expect(ret.objects.length).toEqual(1);
+      expect(ret.objects[0].properties.testProp).toEqual('carrot');
+      expect(ret.objects[0].uuid).toEqual(id);
+    });
+  });
+
   it('should query with hybrid and vector', async () => {
     const ret = await collection.query.hybrid('carrot', {
       limit: 1,

src/collections/query/types.ts

@@ -84,9 +84,15 @@ export type Bm25QueryProperty<T> = {
   weight: number;
 };
 
+export type Bm25OperatorOr = { operator: 'Or'; minimumMatch: number };
+export type Bm25OperatorAnd = { operator: 'And' };
+
+export type Bm25OperatorOptions = Bm25OperatorOr | Bm25OperatorAnd;
+
 export type Bm25SearchOptions<T> = {
   /** Which properties of the collection to perform the keyword search on. */
   queryProperties?: (PrimitiveKeys<T> | Bm25QueryProperty<T>)[];
+  operator?: Bm25OperatorOptions;
 };
 
 /** Base options available in the `query.bm25` method */
@@ -115,6 +121,7 @@ export type HybridSearchOptions<T> = {
   targetVector?: TargetVectorInputType;
   /** The specific vector to search for or a specific vector subsearch. If not specified, the query is vectorized and used in the similarity search. */
   vector?: NearVectorInputType | HybridNearTextSubSearch | HybridNearVectorSubSearch;
+  bm25Operator?: Bm25OperatorOptions;
 };
 
 /** Base options available in the `query.hybrid` method */

src/collections/query/utils.ts

@@ -1,5 +1,5 @@
 import { MultiTargetVectorJoin } from '../index.js';
-import { NearVectorInputType, TargetVectorInputType } from './types.js';
+import { Bm25OperatorOptions, Bm25OperatorOr, NearVectorInputType, TargetVectorInputType } from './types.js';
 
 export class NearVectorInputGuards {
   public static is1DArray(input: NearVectorInputType): input is number[] {
@@ -34,3 +34,13 @@ export class TargetVectorInputGuards {
     return i.combination !== undefined && i.targetVectors !== undefined;
   }
 }
+
+export class Bm25Operator {
+  static and(): Bm25OperatorOptions {
+    return { operator: 'And' };
+  }
+
+  static or(opts: Omit<Bm25OperatorOr, 'operator'>): Bm25OperatorOptions {
+    return { ...opts, operator: 'Or' };
+  }
+}

src/collections/serialize/index.ts

@@ -15,6 +15,8 @@ import {
   NearThermalSearch,
   NearVector,
   NearVideoSearch,
+  SearchOperatorOptions,
+  SearchOperatorOptions_Operator,
   Targets,
   VectorForTarget,
   WeightsForTarget,
@@ -115,6 +117,7 @@ import {
 import {
   BaseHybridOptions,
   BaseNearOptions,
+  Bm25OperatorOptions,
   Bm25Options,
   Bm25QueryProperty,
   Bm25SearchOptions,
@@ -960,10 +963,26 @@ export class Serialize {
     });
   };
 
+  private static bm25SearchOperator = (
+    searchOperator?: Bm25OperatorOptions
+  ): SearchOperatorOptions | undefined => {
+    if (searchOperator) {
+      return SearchOperatorOptions.fromPartial(
+        searchOperator.operator === ('And' as const)
+          ? { operator: SearchOperatorOptions_Operator.OPERATOR_AND }
+          : {
+              operator: SearchOperatorOptions_Operator.OPERATOR_OR,
+              minimumOrTokensMatch: searchOperator.minimumMatch,
+            }
+      );
+    }
+  };
+
   public static bm25Search = <T>(args: { query: string } & Bm25SearchOptions<T>): BM25 => {
     return BM25.fromPartial({
       query: args.query,
       properties: this.bm25QueryProperties(args.queryProperties),
+      searchOperator: this.bm25SearchOperator(args.operator),
     });
   };
 
@@ -1074,6 +1093,7 @@ export class Serialize {
       vectorBytes: vectorBytes,
       vectorDistance: args.maxVectorDistance,
       fusionType: fusionType(args.fusionType),
+      bm25SearchOperator: this.bm25SearchOperator(args.bm25Operator),
       targetVectors,
       targets,
       nearText,