diff --git a/_includes/code/howto/configure-rq/rq-compression-v3.ts b/_includes/code/howto/configure-rq/rq-compression-v3.ts index 59a623c1c..cf09d90d0 100644 --- a/_includes/code/howto/configure-rq/rq-compression-v3.ts +++ b/_includes/code/howto/configure-rq/rq-compression-v3.ts @@ -5,8 +5,9 @@ // ============================== import assert from 'assert'; -import weaviate from 'weaviate-client'; -import { configure } from 'weaviate-client'; +// START EnableRQ // START 1BitEnableRQ // START RQWithOptions // START Uncompressed +import weaviate, { configure } from 'weaviate-client'; +// END EnableRQ // END 1BitEnableRQ // END RQWithOptions // END Uncompressed const client = await weaviate.connectToLocal({ @@ -74,7 +75,8 @@ await client.collections.create({ vectorizers: configure.vectors.text2VecOpenAI({ // highlight-start quantizer: configure.vectorIndex.quantizer.rq({ - bits: 8, // Number of bits + bits: 8, // Optional: Number of bits + rescoreLimit: 20, // Optional: Number of candidates to fetch before rescoring }), // highlight-end }), @@ -84,4 +86,88 @@ await client.collections.create({ }) // END RQWithOptions +// ========================= +// ===== Uncompressed ===== +// ========================= + +client.collections.delete("MyCollection") + +// START Uncompressed + +await client.collections.create({ + name: "MyCollection", + vectorizers: configure.vectors.text2VecOpenAI({ + // highlight-start + quantizer: configure.vectorIndex.quantizer.none() + // highlight-end + }), + properties: [ + { name: "title", dataType: weaviate.configure.dataType.TEXT }, + ], +}) +// END Uncompressed + +// ============================== +// ===== UPDATE SCHEMA ===== +// ============================== + +client.collections.delete("MyCollection") +client.collections.create({ + name: "MyCollection", + vectorizers: configure.vectors.text2VecOpenAI({ + // highlight-start + quantizer: configure.vectorIndex.quantizer.none(), + // highlight-end +}), + properties: [ + { name: "title", dataType: weaviate.configure.dataType.TEXT }, + ], +}) + +// START UpdateSchema // START 1BitUpdateSchema +import { reconfigure } from 'weaviate-client'; + +const collection = client.collections.use("MyCollection") +// END UpdateSchema // END 1BitUpdateSchema +// START UpdateSchema + +await collection.config.update({ + vectorizers: [ reconfigure.vectors.update({ + name: "default", + vectorIndexConfig: reconfigure.vectorIndex.hnsw({ + quantizer: reconfigure.vectorIndex.quantizer.rq(), + }), + })] +}) +// END UpdateSchema + +// ================================ +// ===== UPDATE SCHEMA 1-BIT ===== +// ================================ + +client.collections.delete("MyCollection") +client.collections.create({ + name: "MyCollection", + vectorizers: configure.vectors.text2VecOpenAI({ + // highlight-start + quantizer: configure.vectorIndex.quantizer.none(), + // highlight-end + }), + properties: [ + { name: "title", dataType: weaviate.configure.dataType.TEXT }, + ], +}) + +// START 1BitUpdateSchema + +await collection.config.update({ + vectorizers: [ reconfigure.vectors.update({ + name: "default", + vectorIndexConfig: reconfigure.vectorIndex.hnsw({ + quantizer: reconfigure.vectorIndex.quantizer.rq({ bits: 1 }), + }), + })] +}) +// END 1BitUpdateSchema + client.close() diff --git a/_includes/code/howto/search.bm25.ts b/_includes/code/howto/search.bm25.ts index 3ae89af42..8a621b518 100644 --- a/_includes/code/howto/search.bm25.ts +++ b/_includes/code/howto/search.bm25.ts @@ -8,19 +8,17 @@ import assert from 'assert'; import weaviate from 'weaviate-client'; -const client = await weaviate.connectToWeaviateCloud( - process.env.WEAVIATE_URL, - { - authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY), +const client = await weaviate.connectToWeaviateCloud(process.env.WEAVIATE_URL as string, { + authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY as string), headers: { - 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY, // Replace with your inference API key + 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, // Replace with your inference API key } - } + } ) -// START Basic // START Score // START Properties // START Boost // START Filter // START autocut // START limit +// START BM25GroupBy // START Basic // START Score // START Properties // START Boost // START Filter // START autocut // START limit const jeopardy = client.collections.use('JeopardyQuestion'); -// END Basic // END Score // END Properties // END Boost // END Filter // END autocut // END limit +// END BM25GroupBy // END Basic // END Score // END Properties // END Boost // END Filter // END autocut // END limit // ============================ @@ -250,3 +248,32 @@ for (let object of result.objects) { // assert.deepEqual(result.objects.length, 1); // assert(result.objects[0].properties['answer'].includes('OSHA')); } + +// START BM25GroupBy + +// Grouping parameters +const groupByProperties = { + property: "round", // group by this property + objectsPerGroup: 3, // maximum objects per group + numberOfGroups: 2 // maximum number of groups +} + +// Query +const response = await jeopardy.query.bm25("California", { + groupBy: groupByProperties +}) + +for (let groupName in response.groups) { + console.log(groupName) + // Uncomment to view group objects + // console.log(response.groups[groupName].objects) +} +// END BM25GroupBy + +assert.equal(response.groups <= 2, true) +assert.equal(response.groups > 0, true) + +for (let groupName in response.groups) { + assert.equal(response.groups[groupName].numberOfObjects > 0, true) + assert.equal(response.groups[groupName].numberOfObjects <= 3, true) +} \ No newline at end of file diff --git a/_includes/code/howto/search.filters.ts b/_includes/code/howto/search.filters.ts index f57941ae2..c50b1cd6c 100644 --- a/_includes/code/howto/search.filters.ts +++ b/_includes/code/howto/search.filters.ts @@ -7,10 +7,10 @@ import { vectors, dataType } from 'weaviate-client'; // ===== INSTANTIATION-COMMON ===== // ================================ -// searchMultipleFiltersAnd // searchMultipleFiltersNested +// searchMultipleFiltersAnd // searchMultipleFiltersNested // START ContainsNoneFilter import weaviate, { Filters } from 'weaviate-client'; -// END searchMultipleFiltersAnd // END searchMultipleFiltersNested +// END searchMultipleFiltersAnd // END searchMultipleFiltersNested // END ContainsNoneFilter const client = await weaviate.connectToWeaviateCloud(process.env.WEAVIATE_URL as string, { authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY as string), @@ -20,10 +20,10 @@ const client = await weaviate.connectToWeaviateCloud(process.env.WEAVIATE_URL as } ) -// searchSingleFilter // searchLikeFilter // ContainsAnyFilter // ContainsAllFilter // searchMultipleFiltersNested // searchMultipleFiltersAnd // searchFilterNearText // FilterByPropertyLength // searchCrossReference // searchCrossReference // searchMultipleFiltersNested +// START ContainsNoneFilter // searchSingleFilter // searchLikeFilter // ContainsAnyFilter // ContainsAllFilter // searchMultipleFiltersNested // searchMultipleFiltersAnd // searchFilterNearText // FilterByPropertyLength // searchCrossReference // searchCrossReference // searchMultipleFiltersNested const jeopardy = client.collections.use('JeopardyQuestion'); -// END searchSingleFilter // END searchLikeFilter // END ContainsAnyFilter // END ContainsAllFilter // END searchMultipleFiltersNested // END searchMultipleFiltersAnd // END searchFilterNearText // END FilterByPropertyLength // END searchCrossReference // END searchCrossReference // END searchMultipleFiltersNested +// END ContainsNoneFilter // END searchSingleFilter // END searchLikeFilter // END ContainsAnyFilter // END ContainsAllFilter // END searchMultipleFiltersNested // END searchMultipleFiltersAnd // END searchFilterNearText // END FilterByPropertyLength // END searchCrossReference // END searchCrossReference // END searchMultipleFiltersNested // FilterByTimestamp // filterById const myArticleCollection = client.collections.use('Article'); @@ -175,7 +175,8 @@ const result = await jeopardy.query.fetchObjects({ // highlight-start filters: Filters.and( jeopardy.filter.byProperty('round').equal('Double Jeopardy!'), - jeopardy.filter.byProperty('points').lessThan(600) + jeopardy.filter.byProperty('points').lessThan(600), + Filters.not(jeopardy.filter.byProperty("answer").equal("Yucatan")) ), // highlight-end limit: 3, @@ -261,6 +262,34 @@ for (let object of result.objects) { // } } +// ========================================== +// ===== ContainsNoneFilter ===== +// ========================================== + +// START ContainsNoneFilter + +// highlight-start +const tokenList = ["bird", "animal"] +// highlight-end + +const response = await jeopardy.query.fetchObjects({ + // highlight-start + // Find objects where the `question` property contains none of the strings in `token_list` + filters: jeopardy.filter.byProperty("question").containsNone(tokenList), + // highlight-end + limit: 3 +}) + +for (const object of response.objects) { + console.log(object.properties) +} +// END ContainsNoneFilter + +// Test results +// assert response.objects[0].collection == "JeopardyQuestion" +// assert (token_list[0] not in response.objects[0].properties["question"].lower() and token_list[1] not in response.objects[0].properties["question"].lower()) +// End test + // =================================================== // ===== Filters using Id ===== // =================================================== diff --git a/_includes/code/howto/search.hybrid.ts b/_includes/code/howto/search.hybrid.ts index 117c394cf..f8f02c2a3 100644 --- a/_includes/code/howto/search.hybrid.ts +++ b/_includes/code/howto/search.hybrid.ts @@ -6,7 +6,7 @@ import assert from 'assert'; // ===== INSTANTIATION-COMMON ===== // ================================ -import weaviate from 'weaviate-client'; +import weaviate, { Bm25Operator } from 'weaviate-client'; const client = await weaviate.connectToWeaviateCloud( process.env.WEAVIATE_URL as string, @@ -18,9 +18,9 @@ const client = await weaviate.connectToWeaviateCloud( } ) -// searchHybridBasic // searchHybridWithScore // searchHybridWithAlpha // searchHybridWithFusionType // searchHybridWithProperties // searchHybridWithVector // VectorSimilarity // searchHybridWithFilter // START limit // START autocut // searchHybridWithPropertyWeighting +// START HybridGroupBy // START VectorSimilarityThreshold // START HybridWithBM25OperatorOrWithMin // START HybridWithBM25OperatorAnd // searchHybridBasic // searchHybridWithScore // searchHybridWithAlpha // searchHybridWithFusionType // searchHybridWithProperties // searchHybridWithVector // VectorSimilarity // searchHybridWithFilter // START limit // START autocut // searchHybridWithPropertyWeighting const jeopardy = client.collections.use('JeopardyQuestion'); -// END searchHybridBasic // END searchHybridWithScore // END searchHybridWithAlpha // END searchHybridWithFusionType // END searchHybridWithProperties // END searchHybridWithVector // END searchHybridWithFilter // END VectorSimilarity // END limit // END autocut // END searchHybridWithPropertyWeighting +// END HybridGroupBy // END VectorSimilarityThreshold // END HybridWithBM25OperatorOrWithMin // END HybridWithBM25OperatorAnd // END searchHybridBasic // END searchHybridWithScore // END searchHybridWithAlpha // END searchHybridWithFusionType // END searchHybridWithProperties // END searchHybridWithVector // END searchHybridWithFilter // END VectorSimilarity // END limit // END autocut // END searchHybridWithPropertyWeighting @@ -28,39 +28,39 @@ const jeopardy = client.collections.use('JeopardyQuestion'); // ===== QUERY WITH TARGET VECTOR & Hybrid ===== // =============================================== { - // NamedVectorHybrid - const myNVCollection = client.collections.use('WineReviewNV'); +// NamedVectorHybrid +const myNVCollection = client.collections.use('WineReviewNV'); - const result = await myNVCollection.query.hybrid('a sweet German white wine', { - // highlight-start - targetVector: 'title_country', - // highlight-end - limit: 2, - }) +const result = await myNVCollection.query.hybrid('a sweet German white wine', { + // highlight-start + targetVector: 'title_country', + // highlight-end + limit: 2, +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END NamedVectorHybrid +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END NamedVectorHybrid - // Tests - // assert.deepEqual(result.objects.length, 2); +// Tests +// assert.deepEqual(result.objects.length, 2); } // ============================== // ===== Basic Hybrid Query ===== // ============================== { - // searchHybridBasic +// searchHybridBasic - const result = await jeopardy.query.hybrid('food', { - limit: 2, - }) +const result = await jeopardy.query.hybrid('food', { + limit: 2, +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END searchHybridBasic +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END searchHybridBasic } // Tests // let questionKeys = new Set(Object.keys(result.objects[0].properties)); @@ -72,53 +72,53 @@ const jeopardy = client.collections.use('JeopardyQuestion'); // ===== Hybrid Query with Score ===== // =================================== { - // searchHybridWithScore +// searchHybridWithScore - const result = await jeopardy.query.hybrid('food', { - limit: 2, - returnProperties: ['question', 'answer'], - // highlight-start - returnMetadata: ['score', 'explainScore'] - // highlight-end - }) +const result = await jeopardy.query.hybrid('food', { + limit: 2, + returnProperties: ['question', 'answer'], + // highlight-start + returnMetadata: ['score', 'explainScore'] + // highlight-end +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - console.log(object.metadata?.score); - console.log(object.metadata?.explainScore); - } - // END searchHybridWithScore +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); + console.log(object.metadata?.score); + console.log(object.metadata?.explainScore); +} +// END searchHybridWithScore - // Tests - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer'])); - // const additionalKeys = new Set(Object.keys(result.objects[0].metadata)); - // assert.deepEqual(additionalKeys, new Set(['score', 'explainScore'])); - // assert.equal(result.objects.length, 3); +// Tests +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer'])); +// const additionalKeys = new Set(Object.keys(result.objects[0].metadata)); +// assert.deepEqual(additionalKeys, new Set(['score', 'explainScore'])); +// assert.equal(result.objects.length, 3); } // =================================== // ===== Hybrid Query with Alpha ===== // =================================== { - // searchHybridWithAlpha +// searchHybridWithAlpha - const result = await jeopardy.query.hybrid('food', { - // highlight-start - alpha: 0.25, - // highlight-end - limit: 3 - }) +const result = await jeopardy.query.hybrid('food', { + // highlight-start + alpha: 0.25, + // highlight-end + limit: 3 +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END searchHybridWithAlpha +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END searchHybridWithAlpha - // Tests - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer'])); - // assert.equal(result.objects.length, 3); +// Tests +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer'])); +// assert.equal(result.objects.length, 3); } @@ -126,99 +126,99 @@ const jeopardy = client.collections.use('JeopardyQuestion'); // ===== Hybrid Query with Fusion Methods ===== // ============================================ { - // searchHybridWithFusionType +// searchHybridWithFusionType - const result = await jeopardy.query.hybrid('food', { - limit: 2, - // highlight-start - fusionType: 'RelativeScore', // can also be 'Ranked' - // highlight-end - }) +const result = await jeopardy.query.hybrid('food', { + limit: 2, + // highlight-start + fusionType: 'RelativeScore', // can also be 'Ranked' + // highlight-end +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END searchHybridWithFusionType +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END searchHybridWithFusionType - // Tests - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer'])); - // assert.equal(result.objects.length, 3); +// Tests +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer'])); +// assert.equal(result.objects.length, 3); } // ======================================== // ===== Hybrid Query with Properties ===== // ======================================== { - // searchHybridWithProperties +// searchHybridWithProperties - const result = await jeopardy.query.hybrid('food', { - // highlight-start - queryProperties: ['question'], - // highlight-end - alpha: 0.25, - limit: 3 - }) +const result = await jeopardy.query.hybrid('food', { + // highlight-start + queryProperties: ['question'], + // highlight-end + alpha: 0.25, + limit: 3 +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END searchHybridWithProperties +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END searchHybridWithProperties - // Test - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer'])); - // assert.equal(result.objects.length, 3); +// Test +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer'])); +// assert.equal(result.objects.length, 3); } // ================================================ // ===== Hybrid Query with Property Weighting ===== // ================================================ { - // searchHybridWithPropertyWeighting +// searchHybridWithPropertyWeighting - const result = await jeopardy.query.hybrid('food', { - // highlight-start - queryProperties: ['question^2', 'answer'], - // highlight-end - alpha: 0.25, - limit: 3 - }) +const result = await jeopardy.query.hybrid('food', { + // highlight-start + queryProperties: ['question^2', 'answer'], + // highlight-end + alpha: 0.25, + limit: 3 +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END searchHybridWithPropertyWeighting +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END searchHybridWithPropertyWeighting - // Tests - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer'])); - // assert.equal(result.objects.length, 3); +// Tests +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer'])); +// assert.equal(result.objects.length, 3); } // ==================================== // ===== Hybrid Query with Vector ===== // ==================================== { - // searchHybridWithVector - let queryVector = Array(1536).fill(0.12345) // Query vector [0.12345, 0.12345, 0.12345...] +// searchHybridWithVector +let queryVector = Array(1536).fill(0.12345) // Query vector [0.12345, 0.12345, 0.12345...] - const result = await jeopardy.query.hybrid('food', { - // highlight-start - vector: queryVector, - // highlight-end - alpha: 0.25, - limit: 3 - }) +const result = await jeopardy.query.hybrid('food', { + // highlight-start + vector: queryVector, + // highlight-end + alpha: 0.25, + limit: 3 +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END searchHybridWithVector +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END searchHybridWithVector - // Tests - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer'])); - // assert.equal(result.objects.length, 3); +// Tests +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer'])); +// assert.equal(result.objects.length, 3); } // ========================================= @@ -226,79 +226,79 @@ const jeopardy = client.collections.use('JeopardyQuestion'); // ========================================= { - // VectorSimilarity - - const result = await jeopardy.query.hybrid('California', { - maxVectorDistance: 0.4, // Maximum threshold for the vector search component - vector: { - query: 'large animal', - moveAway: { force: 0.5, concepts: ['mammal', 'terrestrial'] } - }, - alpha: 0.75, - limit: 5, +// VectorSimilarity + +const result = await jeopardy.query.hybrid('California', { + maxVectorDistance: 0.4, // Maximum threshold for the vector search component + vector: { + query: 'large animal', + moveAway: { force: 0.5, concepts: ['mammal', 'terrestrial'] } + }, + alpha: 0.75, + limit: 5, - }) +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END VectorSimilarity +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END VectorSimilarity } // ==================================== // ===== Hybrid Query with Filter ===== // ==================================== { - // searchHybridWithFilter +// searchHybridWithFilter - const result = await jeopardy.query.hybrid('food', { - // highlight-start - filters: jeopardy.filter.byProperty('round').equal('Double Jeopardy!'), - // highlight-end - limit: 3, - }) +const result = await jeopardy.query.hybrid('food', { + // highlight-start + filters: jeopardy.filter.byProperty('round').equal('Double Jeopardy!'), + // highlight-end + limit: 3, +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END searchHybridWithFilter +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END searchHybridWithFilter - // Tests - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer', 'round'])); - // assert.equal(result.objects.length, 3); - // result.objects.map((question) => { - // assert.deepEqual(question.properties.round, 'Double Jeopardy!'); - // console.log(question) - // }) - // for (const question of result.objects) { - // assert.deepEqual(question.round, 'Double Jeopardy!'); - // } +// Tests +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer', 'round'])); +// assert.equal(result.objects.length, 3); +// result.objects.map((question) => { +// assert.deepEqual(question.properties.round, 'Double Jeopardy!'); +// console.log(question) +// }) +// for (const question of result.objects) { +// assert.deepEqual(question.round, 'Double Jeopardy!'); +// } } // =================================== // ===== Hybrid Query with limit ===== // =================================== { - // START limit +// START limit - const result = await jeopardy.query.hybrid('safety', { - // highlight-start - limit: 3, - offset: 1 - // highlight-end - }) +const result = await jeopardy.query.hybrid('safety', { + // highlight-start + limit: 3, + offset: 1 + // highlight-end +}) - for (let object of result.objects) { - console.log(JSON.stringify(object.properties, null, 2)); - } - // END limit +for (let object of result.objects) { + console.log(JSON.stringify(object.properties, null, 2)); +} +// END limit - // Tests - // questionKeys = new Set(Object.keys(result.objects[0].properties)); - // assert.deepEqual(questionKeys, new Set(['question', 'answer'])); - // assert.deepEqual(Object.keys(result.objects[0].metadata), ['score']); - // assert.equal(result.objects.length, 3); - // assert(result.objects[0].properties['answer'].includes('OSHA')); +// Tests +// questionKeys = new Set(Object.keys(result.objects[0].properties)); +// assert.deepEqual(questionKeys, new Set(['question', 'answer'])); +// assert.deepEqual(Object.keys(result.objects[0].metadata), ['score']); +// assert.equal(result.objects.length, 3); +// assert(result.objects[0].properties['answer'].includes('OSHA')); } // ===================================== @@ -332,7 +332,6 @@ const jeopardy = client.collections.use('JeopardyQuestion'); // ========================================= // START VectorSimilarityThreshold -const jeopardy = client.collections.use("JeopardyQuestion") const response = await jeopardy.query.hybrid("California", { alpha: 0.75, @@ -361,8 +360,6 @@ const groupByProperties = { // highlight-end } -const jeopardy = client.collections.use('JeopardyQuestion'); - const response = await jeopardy.query.hybrid('California', { alpha: 0.75, // highlight-start @@ -386,4 +383,52 @@ for (let groupName in response.groups) { assert.equal(response.groups[groupName].numberOfObjects <= 3, true) } +// ======================================== +// ===== Hybrid Query with BM25 Operator (Or) ===== +// ======================================== + +// START HybridWithBM25OperatorOrWithMin + +const response = await jeopardy.query.hybrid("Australian mammal cute", { + // highlight-start + bm25Operator: Bm25Operator.or({ + minimumMatch: 2 + }), + // highlight-end + limit: 3, +}) + +for (const object of response.objects) { + console.log(object.properties) +} +// END HybridWithBM25OperatorOrWithMin + +// Tests +assert.equal(response.objects[0].collection, "JeopardyQuestion") +// End test + + +// ======================================== +// ===== Hybrid Query with BM25 Operator (And) ===== +// ======================================== + +// START HybridWithBM25OperatorAnd + +const response = await jeopardy.query.hybrid( + "Australian mammal cute", { + // highlight-start + bm25Operator: Bm25Operator.and(), // Each result must include all tokens (e.g. "australian", "mammal", "cute") + // highlight-end + limit: 3, +}) + +for (const object of response.objects) { + console.log(object.properties) +} +// END HybridWithBM25OperatorAnd + +// Tests +assert.equal(response.objects[0].collection, "JeopardyQuestion") +// End test + client.close() diff --git a/_includes/code/typescript/howto.configure.rbac.oidc.groups.ts b/_includes/code/typescript/howto.configure.rbac.oidc.groups.ts new file mode 100644 index 000000000..64b8cd740 --- /dev/null +++ b/_includes/code/typescript/howto.configure.rbac.oidc.groups.ts @@ -0,0 +1,402 @@ +import assert from "assert"; +/** + * OIDC Group Management Testing Script with Built-in Keycloak Setup Helper + * Complete example of how to configure RBAC with OIDC groups in Weaviate + */ + +import weaviate, { type WeaviateClient } from "weaviate-client"; + +const { permissions } = weaviate; + +async function testKeycloakConnection(keycloakPorts = [8081]) { + // Try keycloak hostname first (requires /etc/hosts mapping), then localhost + const keycloakConfigs = [ + ['keycloak', 8081], // This should match Weaviate's expected issuer + ['localhost', 8081], // Fallback for initial testing + ]; + + for (const [host, port] of keycloakConfigs) { + const keycloakUrl = `http://${host}:${port}`; + try { + // First check if Keycloak is responding at all + const response = await fetch(keycloakUrl, { + method: 'GET', + signal: AbortSignal.timeout(5000) + }); + + if (response.ok) { + console.log(`OK: Keycloak server found at ${keycloakUrl}`); + + // Check if master realm exists (always exists) + const masterResponse = await fetch( + `${keycloakUrl}/realms/master`, + { signal: AbortSignal.timeout(5000) } + ); + + if (masterResponse.ok) { + console.log(`OK: Keycloak realms accessible`); + + // Check if our test realm exists + const testResponse = await fetch( + `${keycloakUrl}/realms/weaviate-test`, + { signal: AbortSignal.timeout(5000) } + ); + + if (testResponse.ok) { + console.log(`OK: weaviate-test realm found`); + console.log(`OK: weaviate-test realm accessible`); + return keycloakUrl; + } else { + console.log( + `Warning: weaviate-test realm not found - you'll need to create it` + ); + return keycloakUrl; + } + } + } + } catch (e) { + console.log(`Testing ${keycloakUrl}: ${e.message}`); + continue; + } + } + + console.log(`Error: Cannot connect to Keycloak`); + console.log("Hint: Make sure you have '127.0.0.1 keycloak' in /etc/hosts"); + console.log("Hint: Run: echo '127.0.0.1 keycloak' | sudo tee -a /etc/hosts"); + return null; +} + +/** + * Get OIDC token from Keycloak for a user + * @param {Object} params - Token request parameters + * @returns {Promise} Access token if successful, null otherwise + */ +async function getOidcToken({ + keycloakUrl, + clientSecret, + username, + password = 'password123', + realm = 'weaviate-test', + clientId = 'weaviate' +}) { + const tokenUrl = `${keycloakUrl}/realms/${realm}/protocol/openid-connect/token`; + + const data = new URLSearchParams({ + grant_type: 'password', + client_id: clientId, + client_secret: clientSecret, + username: username, + password: password, + }); + + try { + const response = await fetch(tokenUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: data, + signal: AbortSignal.timeout(10000), + }); + + if (response.ok) { + const tokenData = await response.json(); + console.log(`OK: Successfully got token for user: ${username}`); + return tokenData.access_token; + } else { + const responseText = await response.text(); + console.log(`Error: Failed to get token for ${username}: ${response.status}`); + + if (response.status === 401) { + console.log(' → Check username/password or client secret'); + } else if (response.status === 400) { + console.log(' → Check client configuration (Direct Access Grants enabled?)'); + } + console.log(` → Response: ${responseText}`); + return null; + } + } catch (e) { + console.log(`Error: Error getting token for ${username}: ${e.message}`); + return null; + } +} + +/** + * Setup and validate OIDC connection + * @returns {Promise<{clientSecret: string|null, keycloakUrl: string|null}>} + */ +async function setupAndValidateOidc() { + console.log('KEYCLOAK OIDC SETUP VALIDATOR'); + console.log('='.repeat(50)); + + // Test Keycloak connection + console.log('Testing Keycloak connection...'); + const keycloakUrl = await testKeycloakConnection([8081]); + + if (!keycloakUrl) { + console.log('Error: Keycloak not accessible!'); + console.log('\nTroubleshooting:'); + console.log( + "1. Add keycloak to /etc/hosts: echo '127.0.0.1 keycloak' | sudo tee -a /etc/hosts" + ); + console.log('2. Check if docker-compose is running: docker-compose ps'); + console.log('3. Check Keycloak logs: docker-compose logs keycloak'); + return { clientSecret: null, keycloakUrl: null }; + } + + // Check if weaviate-test realm exists + let realmExists = false; + try { + const realmResponse = await fetch( + `${keycloakUrl}/realms/weaviate-test`, + { signal: AbortSignal.timeout(5000) } + ); + realmExists = realmResponse.ok; + } catch (e) { + realmExists = false; + } + + if (!realmExists) { + console.log(`\nWarning: The 'weaviate-test' realm doesn't exist yet.`); + console.log( + 'Please complete the Keycloak setup first with keycloak_helper_script.py, then run this script again.' + ); + return { clientSecret: null, keycloakUrl: null }; + } else { + console.log(`OK: weaviate-test realm accessible`); + console.log('\n' + '-'.repeat(30)); + // Using a fixed secret for automated testing + const clientSecret = 'weaviate-client-secret-123'; + console.log(`Using client secret: ${clientSecret}`); + + // Test tokens with the keycloak_url (which should be http://keycloak:8081) + console.log(`\nTesting OIDC tokens...`); + const adminToken = await getOidcToken({ + keycloakUrl, + clientSecret, + username: 'test-admin' + }); + + if (!adminToken) { + console.log('\nError: Cannot get admin token. Please verify:'); + console.log("- User 'test-admin' exists with password 'password123'"); + console.log("- User is in groups like '/admin-group'"); + console.log("- Client 'weaviate' has 'Direct access grants' enabled"); + console.log('- Client secret is correct'); + return { clientSecret: null, keycloakUrl: null }; + } + + const viewerToken = await getOidcToken({ + keycloakUrl, + clientSecret, + username: 'test-viewer' + }); + + if (!viewerToken) { + console.log('Warning: Viewer token failed, but continuing with admin token'); + } + + console.log('\nOK: OIDC setup validated successfully!'); + return { clientSecret, keycloakUrl }; + } +} + +// Setup and validate OIDC first +const { clientSecret, keycloakUrl } = await setupAndValidateOidc(); + +if (!clientSecret || !keycloakUrl) { + process.exit(1); +} + +console.log('\n' + '='.repeat(60)); +console.log('STARTING OIDC GROUP MANAGEMENT TESTS'); +console.log('='.repeat(60)); + +// The adminClient is used for setup and cleanup that requires root privileges +let adminClient: WeaviateClient; + +// START AdminClient +// Connect to Weaviate as root user (for admin operations) +adminClient = await weaviate.connectToLocal({ + port: 8580, + grpcPort: 50551, + authCredentials: new weaviate.ApiKey("root-user-key"), +}); +// END AdminClient + +// Create test roles for group management +console.log("\nSetting up test roles..."); +const groupPermissions = [ + permissions.collections({ + collection: "TargetCollection*", + read_config: true, + create_collection: true + }), + permissions.data({ + collection: "TargetCollection*", + read: true, + create: true + }), +]; + +await adminClient.roles.delete("testRole"); +await adminClient.roles.create("testRole", groupPermissions); + +await adminClient.roles.delete("groupViewerRole"); +await adminClient.roles.create( + "groupViewerRole", + permissions.data({ collection: "*", read: true }) +); + +console.log("\nADMIN OPERATIONS (Using API Key)"); +console.log('-'.repeat(40)); + +// START AssignOidcGroupRoles +await adminClient.groups.oidc.assignRoles( + "/admin-group", ["testRole", "viewer"] +); +// END AssignOidcGroupRoles +await adminClient.groups.oidc.assignRoles("/viewer-group", ["viewer"]); +await adminClient.groups.oidc.assignRoles( + "/my-test-group", ["groupViewerRole"] +); + +// START GetKnownOidcGroups +const knownGroups = await adminClient.groups.oidc.getKnownGroupNames(); +console.log(`Known OIDC groups (${knownGroups.length}): ${knownGroups}`); +// END GetKnownOidcGroups +assert.strictEqual(knownGroups.length, 3, 'Expected 3 known groups'); +assert.deepStrictEqual( + new Set(knownGroups), + new Set(["/admin-group", "/viewer-group", "/my-test-group"]), + 'Known groups should match expected groups' +); + +// START GetGroupAssignments +const groupAssignments = await adminClient.roles.getGroupAssignments("testRole"); +console.log("Groups assigned to role 'testRole':"); +for (const group of groupAssignments) { + console.log(` - Group ID: ${group.groupID}, Type: ${group.groupType}`); +} +// END GetGroupAssignments +assert.strictEqual(groupAssignments.length, 1, 'Expected 1 group assignment'); +assert.strictEqual(groupAssignments[0].groupID, "/admin-group", 'Group ID should be /admin-group'); + +console.log(`\nOIDC USER OPERATIONS`); +console.log('='.repeat(60)); + +// Get tokens for different users using keycloak_url +// START GetOidcToken +const adminToken = await getOidcToken({ + keycloakUrl: keycloakUrl, + clientSecret: clientSecret, + username: "test-admin" +}); +const viewerToken = await getOidcToken({ + keycloakUrl: keycloakUrl, + clientSecret: clientSecret, + username: "test-viewer" +}); +// END GetOidcToken +assert.notStrictEqual(adminToken, null, 'Admin token should not be null'); +assert.notStrictEqual(viewerToken, null, 'Viewer token should not be null'); + +// --- Admin User Tests --- +// START OidcAdminClient +// Connect as OIDC admin user +const oidcAdminClient = await weaviate.connectToLocal({ + port: 8580, + grpcPort: 50551, + authCredentials: new weaviate.AuthAccessTokenCredentials({ + accessToken: adminToken, + expiresIn: 1000 + }) +}); +// END OidcAdminClient + +// START GetCurrentUserRoles +const myUser = await oidcAdminClient.users.getMyUser(); +const currentRolesList = myUser ? myUser.roles : []; +const roleNames = currentRolesList?.map(role => role.name); +console.log(`Admin user's current roles (${roleNames?.length}): ${roleNames}`); +// END GetCurrentUserRoles +assert.deepStrictEqual( + new Set(roleNames), + new Set(["viewer", "testRole", "groupViewerRole"]), + 'Admin user should have expected roles' +); + +// START GetOidcGroupRoles +const groupRoles = await oidcAdminClient.groups.oidc.getAssignedRoles( + "/admin-group", true +); +console.log(`Roles assigned to '/admin-group': ${Object.keys(groupRoles)}`); +// END GetOidcGroupRoles +assert.deepStrictEqual( + new Set(Object.keys(groupRoles)), + new Set(["testRole", "viewer"]), + 'Admin group should have expected roles' +); +await oidcAdminClient.close(); + +// --- Viewer User Tests --- +// START OidcViewerClient +// Connect as OIDC viewer user +const oidcViewerClient = await weaviate.connectToLocal({ + port: 8580, + grpcPort: 50551, + authCredentials: new weaviate.AuthAccessTokenCredentials({ + accessToken: viewerToken, + expiresIn: 1000 + }), +}); +// END OidcViewerClient + +// START GetCurrentUserRolesViewer +const myNewUser = await oidcViewerClient.users.getMyUser(); +const currentViewerRolesList = myNewUser ? myNewUser.roles : []; +const viewerRoleNames = currentViewerRolesList?.map(role => role.name); +console.log(`Viewer user's current roles (${viewerRoleNames?.length}): ${viewerRoleNames}`); +// END GetCurrentUserRolesViewer +assert.strictEqual(viewerRoleNames?.length, 1, 'Viewer should have 1 role'); +assert.strictEqual(viewerRoleNames[0], "viewer", 'Viewer should have viewer role'); + +// Viewer should have limited permissions but can still see group names +try { + const viewerGroups = await oidcViewerClient.groups.oidc.getKnownGroupNames(); + console.log("Viewer can see groups:", viewerGroups); + assert.deepStrictEqual( + new Set(viewerGroups), + new Set(["/admin-group", "/viewer-group", "/my-test-group"]), + 'Viewer should see all groups' + ); +} catch (e) { + // This part should not be reached if permissions are set correctly + assert.fail(`Viewer user failed to access group operations: ${e}`); +} + +await oidcViewerClient.close(); + +console.log("\nCLEANUP (Admin operations)"); +console.log('-'.repeat(40)); + +// START RevokeOidcGroupRoles +await adminClient.groups.oidc.revokeRoles( + "/admin-group", ["testRole", "viewer"] +); +// END RevokeOidcGroupRoles +await adminClient.groups.oidc.revokeRoles("/viewer-group", ["viewer"]); +await adminClient.groups.oidc.revokeRoles( + "/my-test-group", ["groupViewerRole"] +); + +// Verify cleanup +const finalGroups = await adminClient.groups.oidc.getKnownGroupNames(); +console.log("Remaining known groups after cleanup:", finalGroups); +assert.strictEqual(finalGroups.length, 0, 'All groups should be cleaned up'); + +await adminClient.close(); + +console.log("\n" + '='.repeat(60)); +console.log("OIDC GROUP MANAGEMENT TESTING COMPLETE!"); +console.log('='.repeat(60)); \ No newline at end of file diff --git a/_includes/code/typescript/howto.configure.rbac.roles.ts b/_includes/code/typescript/howto.configure.rbac.roles.ts index 57036af05..6b2d5d93e 100644 --- a/_includes/code/typescript/howto.configure.rbac.roles.ts +++ b/_includes/code/typescript/howto.configure.rbac.roles.ts @@ -213,6 +213,67 @@ if (getNodePermissions) { } +await client.roles.delete("testRole") + +// START AddAliasPermission + +const aliasPermissions = [ + permissions.aliases({ + alias: "TargetAlias*", // Applies to all aliases starting with "TargetAlias" + collection: "TargetCollection*", // Applies to all collections starting with "TargetCollection" + create: true, // Allow alias creation + read: true, // Allow listing aliases + update: true, // Allow updating aliases + delete: false, // Allow deleting aliases + }), +] + +await client.roles.create("testRole", aliasPermissions) +// END AddAliasPermission + +const getAliasPermissions = await client.roles.byName("testRole") + + +if (getAliasPermissions) { + assert.equal((getAliasPermissions.aliasPermissions.some( + permission => permission.alias == "TargetAlias*" + )), true) +} + +client.roles.delete("testRole") + +// START AddReplicationsPermission +// todo +const replicationPermissions = [ + permissions.replicate({ + collection: "TargetCollection*", // Applies to all collections starting with "TargetCollection" + shard: "TargetShard*", // Applies to all shards starting with "TargetShard" + create: true, // Allow replica movement operations + read: true, // Allow retrieving replication status + update: true, // Allow cancelling replication operations + delete: false, // Allow deleting replication operations + }), +] + +await client.roles.create("testRole", replicationPermissions) +// END AddReplicationsPermission + +const getReplicationPermissions = client.roles.byName("testRole") + +// assert any( +// permission.collection == "TargetCollection*" and +// permission.shard == "TargetShard*" +// for permission in permissions.replicate_permissions +// ) + +if (getAliasPermissions) { + assert.equal((getAliasPermissions.aliasPermissions.some( + permission => permission.alias == "TargetAlias*" + )), true) +} + +// todo + await client.roles.delete("testRole") // This is to add additional permission to below diff --git a/_includes/configuration/rq-compression-parameters.mdx b/_includes/configuration/rq-compression-parameters.mdx index 7bee774c6..e8ce8d668 100644 --- a/_includes/configuration/rq-compression-parameters.mdx +++ b/_includes/configuration/rq-compression-parameters.mdx @@ -1,4 +1,4 @@ | Parameter | Type | Default | Details | | :------------------- | :------ | :------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `rq`: `bits` | integer | `8` | The number of bits used to quantize each data point. Can be `8` or `1`.
Learn more about [8-bit](/weaviate/concepts/vector-quantization#8-bit-rq) and [1-bit](/weaviate/concepts/vector-quantization#1-bit-rq) RQ. | +| `rq`: `bits` | integer | `8` | The number of bits used to quantize each data point. Value can be `8` or `1`.

Learn more about [8-bit](/weaviate/concepts/vector-quantization#8-bit-rq) and [1-bit](/weaviate/concepts/vector-quantization#1-bit-rq) RQ. | | `rq`: `rescoreLimit` | integer | `-1` | The minimum number of candidates to fetch before rescoring. | diff --git a/docs/agents/_includes/query_agent.mts b/docs/agents/_includes/query_agent.mts index 2f2eaf25a..e11a94bd7 100644 --- a/docs/agents/_includes/query_agent.mts +++ b/docs/agents/_includes/query_agent.mts @@ -456,7 +456,10 @@ pages.forEach((pageResponse, index) => { // Perform a follow-up query and include the answer from the previous query const basicConversation: ChatMessage[] = [ - { role: "assistant", content: basicResponse.finalAnswer }, + { + role: "assistant", + content: basicResponse.finalAnswer + }, { role: "user", content: "I like the vintage clothes options, can you do the same again but above $200?", @@ -472,8 +475,14 @@ followingResponse.display() // START ConversationalQuery // Create a conversation with multiple turns const conversation: ChatMessage[] = [ - { role: "user", content: "Hi!"}, - { role: "assistant", content: "Hello! How can I assist you today?"}, + { + role: "user", + content: "Hi!" + }, + { + role: "assistant", + content: "Hello! How can I assist you today?" + }, { role: "user", content: "I have some questions about the weather data. You can assume the temperature is in Fahrenheit and the wind speed is in mph.", diff --git a/docs/weaviate/configuration/compression/rq-compression.md b/docs/weaviate/configuration/compression/rq-compression.md index 6d0030b76..929f3b12c 100644 --- a/docs/weaviate/configuration/compression/rq-compression.md +++ b/docs/weaviate/configuration/compression/rq-compression.md @@ -89,6 +89,14 @@ RQ can also be enabled for an existing collection by updating the collection def language="py" /> + + + + + + + + + + ## Additional considerations diff --git a/docs/weaviate/configuration/rbac/manage-groups.mdx b/docs/weaviate/configuration/rbac/manage-groups.mdx index e00c520c8..6b7b7b7c9 100644 --- a/docs/weaviate/configuration/rbac/manage-groups.mdx +++ b/docs/weaviate/configuration/rbac/manage-groups.mdx @@ -10,6 +10,8 @@ import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; import FilteredTextBlock from "@site/src/components/Documentation/FilteredTextBlock"; import OidcGroupPyCode from "!!raw-loader!/_includes/code/python/howto.configure.rbac.oidc.groups.py"; +import OidcGroupTSCode from "!!raw-loader!/_includes/code/typescript/howto.configure.rbac.oidc.groups.ts"; + :::info Added in `v1.33` @@ -39,11 +41,12 @@ This example assigns the `testRole` and `viewer` roles to the `/admin-group`. /> - -```typescript -// TypeScript/JavaScript support coming soon -``` - + @@ -77,11 +80,12 @@ This example removes the `testRole` and `viewer` roles from the `/admin-group`. /> - -```typescript -// TypeScript/JavaScript support coming soon -``` - + @@ -113,11 +117,12 @@ Retrieve a list of all roles that have been assigned to a specific OIDC group. /> - -```typescript -// TypeScript/JavaScript support coming soon -``` - + @@ -158,11 +163,12 @@ This example shows how to get a list of all OIDC groups that Weaviate is aware o /> - -```typescript -// TypeScript/JavaScript support coming soon -``` - + @@ -205,11 +211,12 @@ This example shows which groups have the `testRole` assigned to them. /> - -```typescript -// TypeScript/JavaScript support coming soon -``` - + diff --git a/docs/weaviate/configuration/rbac/manage-roles.mdx b/docs/weaviate/configuration/rbac/manage-roles.mdx index df919d58a..e2e635065 100644 --- a/docs/weaviate/configuration/rbac/manage-roles.mdx +++ b/docs/weaviate/configuration/rbac/manage-roles.mdx @@ -157,28 +157,29 @@ This example creates a role called `testRole` with permissions to: /> - -```typescript -// TS/JS support coming soon -``` - - - - - - - - + + + + + + + + #### Create a role with `Collections` permissions {#collections-permissions} @@ -446,28 +447,29 @@ This example creates a role called `testRole` with permissions to: /> - -```typescript -// TS/JS support coming soon -``` - - - - - - - - + + + + + + + + #### Create a role with `Replications` permissions {#replications-permissions} diff --git a/docs/weaviate/search/bm25.md b/docs/weaviate/search/bm25.md index 43d2e44d5..e2a1781b1 100644 --- a/docs/weaviate/search/bm25.md +++ b/docs/weaviate/search/bm25.md @@ -525,6 +525,15 @@ Define criteria to group search results. /> + + + + - Use `Filters.and` and `Filters.or` methods to combine filters in the JS/TS `v3` API. + Use `Filters.and` and `Filters.or` methods to combine filters in the JS/TS `v3` API. `Filters.not` is used to negate a filter using the logical NOT operator.
These methods take variadic arguments (e.g. `Filters.and(f1, f2, f3, ...)`). To pass an array (e.g. `fs`) as an argument, provide it like so: `Filters.and(...fs)` which will spread the array into its elements. @@ -433,11 +433,12 @@ The `ContainsNone` operator works on text properties and take an array of values /> - -```typescript -// TypeScript/JavaScript support coming soon -``` - + diff --git a/docs/weaviate/search/hybrid.md b/docs/weaviate/search/hybrid.md index c6911d485..e55f55cc1 100644 --- a/docs/weaviate/search/hybrid.md +++ b/docs/weaviate/search/hybrid.md @@ -333,6 +333,14 @@ With the `or` operator, the search returns objects that contain at least `minimu language="python" /> + + + + + +