diff --git a/.vscode/python.code-snippets b/.vscode/python.code-snippets
index 2ace5f2b7a..251778364c 100644
--- a/.vscode/python.code-snippets
+++ b/.vscode/python.code-snippets
@@ -1,27 +1,78 @@
{
- "Weaviate WCD connect": {
- "prefix": "Weaviate-wcd-connect",
+ "Weaviate Starter Code - Local": {
+ "prefix": "weaviate-starter-local",
+ "description": "Create a new Weaviate collection: local w/ Ollama",
"body": [
"import weaviate",
+ "from weaviate.classes.config import Configure, Property, DataType",
+ "",
+ "client = weaviate.connect_to_local()",
+ "",
+ "collection_name = \"${1:TempCollection}\"",
+ "",
+ "client.collections.delete(collection_name)",
+ "",
+ "client.collections.create(",
+ " collection_name,",
+ " properties=[",
+ " Property(\"${2:title}\", DataType.TEXT),",
+ " Property(\"${3:body}\", DataType.TEXT),${4}",
+ " ],",
+ " vectorizer_config=[",
+ " Configure.NamedVectors.text2vec_ollama(",
+ " name=\"default\",",
+ " source_properties=[\"${2:title}\", \"${3:body}\"],",
+ " api_endpoint=\"${5:http://host.docker.internal:11434}\",",
+ " model=\"${6:snowflake-arctic-embed}\",",
+ " )",
+ " ],",
+ " generative_config=Configure.Generative.text2vec_ollama(",
+ " api_endpoint=\"${5:http://host.docker.internal:11434}\",",
+ " model=\"${7:llama3.2}\",",
+ " )",
+ ")"
+ ]
+ },
+ "Weaviate Starter Code - WCD": {
+ "prefix": "weaviate-starter-wcd",
+ "description": "Create a new Weaviate collection: WCD w/ Cohere",
+ "body": [
+ "import weaviate",
+ "from weaviate.classes.config import Configure, Property, DataType",
"from weaviate.classes.init import Auth",
"import os",
"",
- "wcd_url = os.environ[\"WCD_DEMO_URL\"]",
- "wcd_api_key = os.environ[\"WCD_DEMO_RO_KEY\"]",
- "openai_api_key = os.environ[\"OPENAI_APIKEY\"]",
- "cohere_api_key = os.environ[\"COHERE_APIKEY\"]",
+ "cohere_key = os.getenv(\"COHERE_APIKEY\")",
+ "weaviate_url = os.getenv(\"WEAVIATE_URL\")",
+ "weaviate_key = os.getenv(\"WEAVIATE_API_KEY\")",
"",
"client = weaviate.connect_to_weaviate_cloud(",
- " cluster_url=wcd_url,",
- " auth_credentials=Auth.api_key(wcd_api_key),",
- " headers={",
- " \"X-OpenAI-Api-Key\": openai_api_key,",
- " \"X-Cohere-Api-Key\": cohere_api_key,",
- " },",
+ " cluster_url=weaviate_url,",
+ " auth_credentials=Auth.api_key(weaviate_key),",
+ " headers={",
+ " \"X-Cohere-Api-Key\": cohere_key,",
+ " }",
")",
- ""
- ],
- "description": "Connect to WCD edu demo instance"
+ "",
+ "collection_name = \"${1:TempCollection}\"",
+ "",
+ "client.collections.delete(collection_name)",
+ "",
+ "client.collections.create(",
+ " collection_name,",
+ " properties=[",
+ " Property(\"${2:title}\", DataType.TEXT),",
+ " Property(\"${3:body}\", DataType.TEXT),${4}",
+ " ],",
+ " vectorizer_config=[",
+ " Configure.NamedVectors.text2vec_cohere(",
+ " name=\"default\",",
+ " source_properties=[\"${2:title}\", \"${3:body}\"]",
+ " )",
+ " ],",
+ " generative_config=Configure.Generative.cohere()",
+ ")"
+ ]
},
"Code snippet section": {
"prefix": "code-section",
diff --git a/README.md b/README.md
index a36f9ca2c6..633d04c723 100644
--- a/README.md
+++ b/README.md
@@ -12,11 +12,7 @@ first.
### Node.js Installation
Use the [nvm](https://github.com/nvm-sh/nvm) package manager to install Node.js.
-The `nvm` project page provides an installation script.
-
-```
-curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
-```
+The `nvm` project page provides an [installation script](https://github.com/nvm-sh/nvm?tab=readme-ov-file#installing-and-updating).
After you install `nvm`, use `nvm` to install Node.js.
@@ -24,13 +20,11 @@ After you install `nvm`, use `nvm` to install Node.js.
nvm install
```
-By default, `nvm` installs the most recent version of Node.js. Install Node.js
-19.9.0 as well. Version 19.9.0 is more compatible with the current
-`weaviate.io` project dependencies.
+By default, `nvm` installs the most recent version of Node.js. Also install the version of Node.js that is specified in `.github/workflows/pull_requests.yaml`. At the time of writing it is version 20.
```
-nvm install 19.9.0
-nvm use 19.9.0
+nvm install 20
+nvm use 20
```
### yarn Installation
@@ -105,7 +99,7 @@ the other project dependencies.
Switch to the project directory, then use yarn to update the dependencies.
```
-cd weaviate.io
+cd weaviate-io
yarn install
```
@@ -116,23 +110,10 @@ You may see some warnings during the installation.
When the installation completes, start the `yarn` server to test your build.
```
-yarn start &
+yarn start
```
-`yarn` builds the project as a static web site and starts a server to host it.
-`yarn` also opens a browser window connected to http://localhost:3000/ where
-you can see your changes.
-
-Most changes are reflected live without having to restart the server.
-
-If you run ``yarn start`` in the foreground (without the "&"), you have to open
-a second terminal to continue working on the command line. When you open a
-second terminal, be sure to set the correct Node.js version before running
-additional `yarn` commands.
-
-```
-nvm use node 19.9.0
-```
+This will build the site and start a local server, then open http://localhost:3000/ showing the local build. If you close the terminal, the server will stop. Or press `Ctrl+C`/`Cmd+C` to stop the server.
### Build the Web Site
@@ -178,7 +159,7 @@ If you are using GitHub pages for hosting, this command is a convenient way to b
Code examples in the documentation are in one of two formats:
-#### New format
+#### Extracted from scripts
In many files, you will see a format similar to:
@@ -204,7 +185,7 @@ import TSCode from '!!raw-loader!/_includes/code/howto/manage-data.create.ts';
text={TSCode}
startMarker="// ValidateObject START"
endMarker="// ValidateObject END"
- language="ts"
+ language="tsv2"
/>
@@ -216,7 +197,7 @@ Here, the `FilteredTextBlock` component loads lines between the `startMarker` an
For more information about tests, please see [README-tests.md](./README-tests.md).
-#### Legacy format
+#### Pure text
In some code examples, the code will be written directly inside the `TabItem` component, as shown below.
diff --git a/_build_scripts/slack-find-author.sh b/_build_scripts/slack-find-author.sh
index 113134bccc..98b834f0dd 100644
--- a/_build_scripts/slack-find-author.sh
+++ b/_build_scripts/slack-find-author.sh
@@ -4,6 +4,7 @@ set -e
# A map of the github contributors that connects to @Slack handles
declare -A git_slack_map
git_slack_map=(
+ ["Abdel Rodríguez"]="<@U03R5ELDHHB>"
["Andrzej Liszka"]="<@U036Y4GPB6W>"
["Bob van Luijt"]="<@U6P955HC4>"
["Charlie Harr"]="<@U044XTHRVFA>"
@@ -26,15 +27,19 @@ git_slack_map=(
["m-newhauser"]="<@U07K9AJCG2F>"
["Marcin Antas"]="<@U01E5BJ3UV7>"
["Mohamed Shahin"]="<@U05V4HPJ3M0>"
+ ["Nate Wilkinson"]="<@U06SCMA8ZB9>"
["Parker Duckworth"]="<@U034QPLGSCU>"
["Peter Schramm"]="<@U03MWHJQ7PX>"
["Philip Vollet"]="<@U0573N5V97A>"
["Sebastian Witalec"]="<@U03DENV56CR>"
["Shan-Weaviate"]="<@U05DKAH2ZL1>"
+ ["Spiros"]="<@U07G6HDV0HK>"
["Stefan Bogdan"]="<@U01DPKTVA93>"
["Svitlana"]="<@U03DQTXFDHS>"
["svitlana-sm"]="<@U03DQTXFDHS>"
["thomashacker"]="<@U056E1ZEM3L>"
+ ["TuanaCelik"]="<@U07TUMJ1SUS>"
+ ["Victoria Slocum"]="<@U05K0QFGRGV>"
["Wera"]="<@U043TKSEU5V>"
["Zain Hasan"]="<@U043TKSJQF9>"
["Mohamed Shahin"]="<@U05V4HPJ3M0>"
diff --git a/_build_scripts/update-config-versions.js b/_build_scripts/update-config-versions.js
index add3353325..31aa4f392a 100644
--- a/_build_scripts/update-config-versions.js
+++ b/_build_scripts/update-config-versions.js
@@ -1,30 +1,53 @@
const fetch = require('node-fetch');
const getRepoVersion = async (repoName) => {
- const response = await fetch( // fetch all release versions
- `https://api.github.com/repos/weaviate/${repoName}/releases`,
- {
- method: 'GET',
- headers: {
- 'Content-Type': 'application/json',
- 'authorization': // Use the github token if available
- (process.env.GH_API_TOKEN) ?
- `Bearer ${ process.env.GH_API_TOKEN }` : ''
+ try {
+ const response = await fetch( // fetch all release versions
+ `https://api.github.com/repos/weaviate/${repoName}/releases`,
+ {
+ method: 'GET',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'authorization': // Use the github token if available
+ (process.env.GH_API_TOKEN) ?
+ `Bearer ${ process.env.GH_API_TOKEN }` : ''
+ }
}
+ );
+
+ // First check if the response was ok
+ if (!response.ok) {
+ throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
}
- );
- const releases = await response.json();
- const highestVersion = releases
- .filter(item => !item.prerelease) // remove pre-release items
- .map(item => item.tag_name) // keep only the tag_name
- .sort() // sort items alphabetically – ascending
- .pop() // the last item contains the highest version (what we need)
- .replace('v', '') // remove the v from the name "v1.26.1" => "1.26.1"
+ const releases = await response.json();
+
+ // Check if releases is actually an array
+ if (!Array.isArray(releases)) {
+ // Debug log to see what we're getting
+ console.log(`Raw response for ${repoName}:`, JSON.stringify(releases).slice(0, 200));
+ console.error(`Unexpected response format for ${repoName}:`, releases);
+ throw new Error(`Expected array of releases but got ${typeof releases}`);
+ }
+
+ if (releases.length === 0) {
+ throw new Error(`No releases found for ${repoName}`);
+ }
- console.log(`${repoName} ${highestVersion}`)
+ const highestVersion = releases
+ .filter(item => !item.prerelease) // remove pre-release items
+ .map(item => item.tag_name) // keep only the tag_name
+ .sort() // sort items alphabetically – ascending
+ .pop() // the last item contains the highest version (what we need)
+ .replace('v', '') // remove the v from the name "v1.26.1" => "1.26.1"
- return highestVersion;
+ console.log(`${repoName} ${highestVersion}`);
+ return highestVersion;
+ } catch (error) {
+ console.error(`Error fetching version for ${repoName}:`, error);
+ // Maybe return a default version or rethrow depending on your needs
+ throw error;
+ }
}
// Build time versions replace values set in versions-config.json
@@ -63,4 +86,4 @@ const updateConfigFile = async () => {
});
}
-updateConfigFile();
\ No newline at end of file
+updateConfigFile();
diff --git a/_includes/ann-deep-96.mdx b/_includes/ann-deep-96.mdx
deleted file mode 100644
index dad80423b8..0000000000
--- a/_includes/ann-deep-96.mdx
+++ /dev/null
@@ -1,80 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-
-
-
-|efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 64 | 16 | 64 | **94.44%** | **9301** | 310 | 3.14ms | 7.21ms | 3305s |
-| 128 | 16 | 64 | **96.06%** | **8957** | 299 | 3.28ms | 7.24ms | 3804s |
-| 64 | 64 | 64 | **96.84%** | **8760** | 292 | 3.36ms | 6.97ms | 3253s |
-| 128 | 32 | 64 | **97.88%** | **8473** | 282 | 3.48ms | 7.4ms | 3533s |
-| 128 | 64 | 64 | **98.27%** | **7984** | 266 | 3.66ms | 7.52ms | 3631s |
-| 256 | 32 | 64 | **98.78%** | **7916** | 264 | 3.71ms | 7.83ms | 4295s |
-| 512 | 32 | 64 | **98.95%** | **7876** | 263 | 3.73ms | 7.47ms | 5477s |
-| 256 | 64 | 64 | **99.06%** | **7839** | 261 | 3.75ms | 7.21ms | 4392s |
-| 512 | 64 | 64 | **99.32%** | **7238** | 241 | 4.05ms | 7.67ms | 6039s |
-| 256 | 64 | 128 | **99.42%** | **5767** | 192 | 5.1ms | 8.39ms | 4392s |
-| 512 | 64 | 128 | **99.52%** | **5509** | 184 | 5.34ms | 8.7ms | 6039s |
-| 256 | 32 | 256 | **99.66%** | **4672** | 156 | 6.32ms | 10.11ms | 4295s |
-| 512 | 32 | 256 | **99.82%** | **4467** | 149 | 6.62ms | 10.29ms | 5477s |
-| 512 | 64 | 256 | **99.9%** | **3683** | 123 | 7.97ms | 12.72ms | 6039s |
-| 512 | 32 | 512 | **99.94%** | **2842** | 95 | 10.37ms | 15.25ms | 5477s |
-| 512 | 64 | 512 | **99.95%** | **2288** | 76 | 12.84ms | 20.72ms | 6039s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 64 | 16 | 64 | **91.58%** | **8679** | 289 | 3.35ms | 7.3ms | 3305s |
-| 128 | 16 | 64 | **93.68%** | **8402** | 280 | 3.47ms | 6.9ms | 3804s |
-| 64 | 32 | 64 | **94.11%** | **8255** | 275 | 3.55ms | 7.61ms | 3275s |
-| 64 | 64 | 64 | **94.67%** | **8184** | 273 | 3.58ms | 7.19ms | 3253s |
-| 128 | 64 | 64 | **96.95%** | **7575** | 253 | 3.88ms | 7.79ms | 3631s |
-| 256 | 32 | 64 | **97.53%** | **7539** | 251 | 3.87ms | 7.81ms | 4295s |
-| 512 | 32 | 64 | **97.92%** | **7399** | 247 | 3.96ms | 8.04ms | 5477s |
-| 256 | 64 | 64 | **98.15%** | **7287** | 243 | 4.02ms | 7.3ms | 4392s |
-| 512 | 64 | 64 | **98.76%** | **6838** | 228 | 4.27ms | 7.96ms | 6039s |
-| 256 | 64 | 128 | **98.77%** | **5658** | 189 | 5.2ms | 8.7ms | 4392s |
-| 512 | 64 | 128 | **99.23%** | **5233** | 174 | 5.62ms | 9.25ms | 6039s |
-| 256 | 32 | 256 | **99.44%** | **4454** | 148 | 6.58ms | 10.11ms | 4295s |
-| 512 | 32 | 256 | **99.61%** | **4270** | 142 | 6.89ms | 10.77ms | 5477s |
-| 512 | 64 | 256 | **99.78%** | **3534** | 118 | 8.26ms | 12.97ms | 6039s |
-| 256 | 32 | 512 | **99.8%** | **2932** | 98 | 10.04ms | 14.79ms | 4295s |
-| 512 | 32 | 512 | **99.88%** | **2767** | 92 | 10.64ms | 15.67ms | 5477s |
-| 512 | 64 | 512 | **99.93%** | **2233** | 74 | 13.12ms | 21.24ms | 6039s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 512 | 8 | 64 | **72.88%** | **4734** | 158 | 6.06ms | 9.94ms | 4327s |
-| 64 | 16 | 64 | **82.08%** | **4645** | 155 | 6.25ms | 9.31ms | 3305s |
-| 512 | 16 | 64 | **85.81%** | **4556** | 152 | 6.33ms | 9.56ms | 4922s |
-| 64 | 32 | 64 | **86.23%** | **4492** | 150 | 6.43ms | 9.82ms | 3275s |
-| 64 | 64 | 64 | **87.25%** | **4488** | 150 | 6.45ms | 9.36ms | 3253s |
-| 64 | 32 | 128 | **89.05%** | **4347** | 145 | 6.67ms | 10.05ms | 3275s |
-| 512 | 16 | 128 | **89.08%** | **4347** | 145 | 6.65ms | 10.31ms | 4922s |
-| 64 | 64 | 128 | **89.88%** | **4284** | 143 | 6.78ms | 9.86ms | 3253s |
-| 256 | 32 | 64 | **91.99%** | **4146** | 138 | 7.01ms | 10.36ms | 4295s |
-| 512 | 32 | 64 | **92.7%** | **4092** | 136 | 7.08ms | 10.33ms | 5477s |
-| 256 | 64 | 64 | **93.85%** | **3917** | 131 | 7.39ms | 10.68ms | 4392s |
-| 256 | 32 | 128 | **94.22%** | **3913** | 130 | 7.43ms | 10.74ms | 4295s |
-| 512 | 32 | 128 | **94.83%** | **3856** | 129 | 7.54ms | 11.08ms | 5477s |
-| 512 | 64 | 64 | **95.14%** | **3816** | 127 | 7.6ms | 11.23ms | 6039s |
-| 256 | 64 | 128 | **95.65%** | **3688** | 123 | 7.9ms | 11.12ms | 4392s |
-| 128 | 32 | 256 | **96.9%** | **3317** | 111 | 8.78ms | 12.5ms | 3533s |
-| 256 | 32 | 256 | **97.91%** | **3182** | 106 | 9.19ms | 12.91ms | 4295s |
-| 512 | 32 | 256 | **98.29%** | **3090** | 103 | 9.48ms | 13.16ms | 5477s |
-| 256 | 64 | 256 | **98.48%** | **2896** | 97 | 10.1ms | 14.27ms | 4392s |
-| 512 | 64 | 256 | **99.02%** | **2707** | 90 | 10.78ms | 15.47ms | 6039s |
-| 256 | 32 | 512 | **99.34%** | **2310** | 77 | 12.65ms | 17.56ms | 4295s |
-| 512 | 32 | 512 | **99.52%** | **2200** | 73 | 13.27ms | 18.76ms | 5477s |
-| 256 | 64 | 512 | **99.53%** | **2032** | 68 | 14.3ms | 21.44ms | 4392s |
-| 512 | 64 | 512 | **99.75%** | **1879** | 63 | 15.56ms | 23.65ms | 6039s |
-
-
-
diff --git a/_includes/ann-gist-960.mdx b/_includes/ann-gist-960.mdx
deleted file mode 100644
index e22c5d0def..0000000000
--- a/_includes/ann-gist-960.mdx
+++ /dev/null
@@ -1,75 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-
-
-
-|efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 64 | 8 | 64 | **66.6%** | **2759** | 92 | 10.59ms | 13.77ms | 1832s |
-| 128 | 8 | 64 | **70.7%** | **2734** | 91 | 10.7ms | 13.97ms | 1861s |
-| 512 | 8 | 64 | **75.0%** | **2724** | 91 | 10.78ms | 14.87ms | 2065s |
-| 64 | 16 | 64 | **79.8%** | **2618** | 87 | 11.04ms | 14.69ms | 1838s |
-| 128 | 16 | 64 | **83.9%** | **2577** | 86 | 11.21ms | 15.55ms | 1904s |
-| 256 | 16 | 64 | **87.1%** | **2518** | 84 | 11.54ms | 14.49ms | 2016s |
-| 128 | 32 | 64 | **89.6%** | **2425** | 81 | 11.85ms | 15.37ms | 1931s |
-| 256 | 32 | 64 | **92.6%** | **2388** | 80 | 12.09ms | 15.99ms | 2074s |
-| 256 | 64 | 64 | **94.1%** | **2207** | 74 | 13.08ms | 18.56ms | 2130s |
-| 512 | 32 | 64 | **94.6%** | **2073** | 69 | 14.11ms | 17.37ms | 2361s |
-| 512 | 32 | 128 | **96.2%** | **1985** | 66 | 14.67ms | 19.32ms | 2361s |
-| 512 | 64 | 64 | **96.2%** | **1951** | 65 | 14.7ms | 19.61ms | 2457s |
-| 512 | 16 | 256 | **96.2%** | **1839** | 61 | 15.9ms | 19.84ms | 2217s |
-| 512 | 64 | 128 | **96.7%** | **1603** | 53 | 18.06ms | 24.44ms | 2457s |
-| 512 | 32 | 256 | **98.7%** | **1514** | 50 | 19.16ms | 24.43ms | 2361s |
-| 512 | 32 | 512 | **99.1%** | **999** | 33 | 29.12ms | 38.89ms | 2361s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 128 | 8 | 64 | **65.88%** | **2649** | 88 | 11.02ms | 14.89ms | 1861s |
-| 512 | 8 | 64 | **69.68%** | **2625** | 88 | 11.03ms | 15.08ms | 2065s |
-| 64 | 16 | 64 | **74.17%** | **2557** | 85 | 11.29ms | 15.54ms | 1838s |
-| 128 | 16 | 64 | **80.23%** | **2518** | 84 | 11.5ms | 15.54ms | 1904s |
-| 64 | 64 | 64 | **81.98%** | **2387** | 80 | 12.11ms | 16.31ms | 1860s |
-| 256 | 16 | 64 | **82.87%** | **2355** | 79 | 12.37ms | 17.67ms | 2016s |
-| 128 | 64 | 64 | **88.11%** | **2312** | 77 | 12.56ms | 16.57ms | 1952s |
-| 256 | 32 | 64 | **89.83%** | **2297** | 77 | 12.55ms | 18.98ms | 2074s |
-| 512 | 32 | 64 | **91.85%** | **2002** | 67 | 14.47ms | 20.03ms | 2361s |
-| 256 | 64 | 64 | **92.04%** | **1937** | 65 | 14.93ms | 21.48ms | 2130s |
-| 512 | 32 | 128 | **94.14%** | **1935** | 65 | 15.05ms | 19.86ms | 2361s |
-| 512 | 64 | 64 | **94.72%** | **1860** | 62 | 15.56ms | 21.91ms | 2457s |
-| 512 | 64 | 128 | **95.99%** | **1569** | 52 | 18.38ms | 24.81ms | 2457s |
-| 256 | 32 | 256 | **96.48%** | **1556** | 52 | 18.6ms | 24.68ms | 2074s |
-| 512 | 32 | 256 | **97.76%** | **1483** | 49 | 19.53ms | 25.24ms | 2361s |
-| 512 | 64 | 256 | **98.62%** | **1286** | 43 | 22.3ms | 30.06ms | 2457s |
-| 512 | 32 | 512 | **99.16%** | **981** | 33 | 29.53ms | 37.97ms | 2361s |
-| 512 | 64 | 512 | **99.47%** | **880** | 29 | 32.45ms | 44.66ms | 2457s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 512 | 8 | 64 | **56.05%** | **1997** | 67 | 14.5ms | 20.28ms | 2065s |
-| 256 | 8 | 128 | **60.26%** | **1945** | 65 | 14.66ms | 18.39ms | 1938s |
-| 64 | 16 | 64 | **61.8%** | **1862** | 62 | 15.42ms | 20.05ms | 1838s |
-| 128 | 16 | 64 | **68.05%** | **1832** | 61 | 15.61ms | 20.05ms | 1904s |
-| 512 | 16 | 128 | **77.53%** | **1802** | 60 | 16.1ms | 19.07ms | 2217s |
-| 128 | 64 | 64 | **78.26%** | **1744** | 58 | 16.59ms | 21.48ms | 1952s |
-| 128 | 32 | 128 | **79.71%** | **1713** | 57 | 16.68ms | 21.37ms | 1931s |
-| 256 | 32 | 64 | **80.3%** | **1652** | 55 | 17.49ms | 23.8ms | 2074s |
-| 512 | 32 | 128 | **86.91%** | **1624** | 54 | 17.83ms | 23.28ms | 2361s |
-| 512 | 16 | 256 | **88.31%** | **1515** | 51 | 19.08ms | 24.64ms | 2217s |
-| 128 | 32 | 256 | **89.11%** | **1477** | 49 | 19.72ms | 25.54ms | 1931s |
-| 256 | 32 | 256 | **92.63%** | **1361** | 45 | 21.34ms | 28.19ms | 2074s |
-| 512 | 32 | 256 | **94.49%** | **1308** | 44 | 22.17ms | 29.1ms | 2361s |
-| 512 | 64 | 256 | **96.44%** | **1152** | 38 | 24.88ms | 33.15ms | 2457s |
-| 256 | 32 | 512 | **96.94%** | **1001** | 33 | 28.71ms | 36.11ms | 2074s |
-| 256 | 64 | 512 | **97.87%** | **893** | 30 | 31.91ms | 42.6ms | 2130s |
-| 512 | 32 | 512 | **98.04%** | **870** | 29 | 32.84ms | 42.31ms | 2361s |
-| 512 | 64 | 512 | **98.8%** | **812** | 27 | 34.96ms | 47.45ms | 2457s |
-
-
-
diff --git a/_includes/ann-glove-25.mdx b/_includes/ann-glove-25.mdx
deleted file mode 100644
index 3c797af655..0000000000
--- a/_includes/ann-glove-25.mdx
+++ /dev/null
@@ -1,79 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-
-
-
-|efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 128 | 8 | 64 | **94.47%** | **19752** | 658 | 1.48ms | 2.49ms | 178s |
-| 512 | 8 | 64 | **95.3%** | **19704** | 657 | 1.48ms | 2.61ms | 272s |
-| 512 | 16 | 64 | **99.26%** | **17583** | 586 | 1.65ms | 2.82ms | 308s |
-| 256 | 16 | 64 | **99.27%** | **17177** | 573 | 1.7ms | 2.67ms | 232s |
-| 128 | 32 | 64 | **99.72%** | **15443** | 515 | 1.9ms | 2.83ms | 184s |
-| 256 | 32 | 64 | **99.84%** | **15187** | 506 | 1.93ms | 2.82ms | 261s |
-| 512 | 32 | 64 | **99.89%** | **14401** | 480 | 2.04ms | 2.93ms | 354s |
-| 256 | 64 | 64 | **99.9%** | **13490** | 450 | 2.17ms | 3.17ms | 276s |
-| 512 | 64 | 64 | **99.96%** | **12626** | 421 | 2.32ms | 3.37ms | 388s |
-| 512 | 64 | 128 | **99.98%** | **8665** | 289 | 3.38ms | 4.82ms | 388s |
-| 256 | 32 | 256 | **99.98%** | **7191** | 240 | 4.07ms | 5.77ms | 261s |
-| 128 | 64 | 256 | **99.99%** | **6958** | 232 | 4.18ms | 6.17ms | 195s |
-| 512 | 32 | 256 | **100.0%** | **6694** | 223 | 4.39ms | 5.99ms | 354s |
-| 128 | 32 | 512 | **100.0%** | **4568** | 152 | 6.4ms | 9.27ms | 184s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 128 | 8 | 64 | **91.33%** | **16576** | 553 | 1.75ms | 2.82ms | 178s |
-| 256 | 8 | 64 | **91.98%** | **16474** | 549 | 1.76ms | 2.87ms | 205s |
-| 512 | 8 | 64 | **92.13%** | **16368** | 546 | 1.77ms | 2.85ms | 272s |
-| 64 | 16 | 64 | **96.56%** | **15003** | 500 | 1.93ms | 2.94ms | 160s |
-| 512 | 16 | 64 | **97.95%** | **14996** | 500 | 1.92ms | 2.78ms | 308s |
-| 64 | 64 | 64 | **98.04%** | **14197** | 473 | 2.05ms | 3.14ms | 167s |
-| 128 | 32 | 64 | **99.06%** | **13482** | 449 | 2.17ms | 3.07ms | 184s |
-| 256 | 32 | 64 | **99.44%** | **13237** | 441 | 2.2ms | 3.22ms | 261s |
-| 512 | 32 | 64 | **99.56%** | **12661** | 422 | 2.31ms | 3.32ms | 354s |
-| 256 | 64 | 64 | **99.63%** | **12014** | 400 | 2.43ms | 3.37ms | 276s |
-| 512 | 64 | 64 | **99.76%** | **11300** | 377 | 2.58ms | 3.56ms | 388s |
-| 512 | 32 | 128 | **99.76%** | **9365** | 312 | 3.14ms | 4.73ms | 354s |
-| 256 | 64 | 128 | **99.79%** | **8669** | 289 | 3.34ms | 4.67ms | 276s |
-| 512 | 64 | 128 | **99.89%** | **7990** | 266 | 3.65ms | 5.09ms | 388s |
-| 256 | 32 | 256 | **99.95%** | **6771** | 226 | 4.32ms | 5.84ms | 261s |
-| 512 | 32 | 256 | **99.97%** | **6286** | 210 | 4.66ms | 6.33ms | 354s |
-| 512 | 64 | 256 | **99.99%** | **5225** | 174 | 5.55ms | 8.11ms | 388s |
-| 256 | 32 | 512 | **100.0%** | **4281** | 143 | 6.84ms | 9.55ms | 261s |
-| 512 | 32 | 512 | **100.0%** | **3917** | 131 | 7.47ms | 10.33ms | 354s |
-| 256 | 64 | 512 | **100.0%** | **3611** | 120 | 8.03ms | 12.03ms | 276s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 64 | 8 | 64 | **78.16%** | **6202** | 207 | 4.55ms | 6.21ms | 152s |
-| 256 | 8 | 64 | **80.07%** | **6044** | 201 | 4.59ms | 8.59ms | 205s |
-| 64 | 8 | 128 | **81.93%** | **5968** | 199 | 4.73ms | 6.98ms | 152s |
-| 512 | 16 | 64 | **91.28%** | **5930** | 198 | 4.75ms | 6.86ms | 308s |
-| 64 | 64 | 64 | **92.52%** | **5768** | 192 | 4.91ms | 6.38ms | 167s |
-| 128 | 16 | 128 | **93.17%** | **5650** | 188 | 5.02ms | 6.47ms | 185s |
-| 128 | 32 | 64 | **94.91%** | **5543** | 185 | 5.13ms | 6.81ms | 184s |
-| 256 | 32 | 64 | **96.07%** | **5524** | 184 | 5.12ms | 6.71ms | 261s |
-| 512 | 32 | 64 | **96.45%** | **5321** | 177 | 5.32ms | 7.51ms | 354s |
-| 128 | 32 | 128 | **96.54%** | **5254** | 175 | 5.42ms | 7.01ms | 184s |
-| 256 | 32 | 128 | **97.48%** | **5235** | 175 | 5.43ms | 7.34ms | 261s |
-| 512 | 32 | 128 | **97.79%** | **5045** | 168 | 5.65ms | 7.15ms | 354s |
-| 256 | 64 | 128 | **98.21%** | **4889** | 163 | 5.86ms | 7.75ms | 276s |
-| 512 | 64 | 128 | **98.75%** | **4667** | 156 | 6.13ms | 7.85ms | 388s |
-| 128 | 32 | 256 | **99.01%** | **4298** | 143 | 6.71ms | 8.76ms | 184s |
-| 256 | 32 | 256 | **99.43%** | **4242** | 141 | 6.77ms | 8.74ms | 261s |
-| 512 | 32 | 256 | **99.57%** | **4069** | 136 | 7.1ms | 9.01ms | 354s |
-| 256 | 64 | 256 | **99.61%** | **3854** | 128 | 7.47ms | 10.13ms | 276s |
-| 512 | 64 | 256 | **99.79%** | **3634** | 121 | 7.92ms | 10.88ms | 388s |
-| 256 | 32 | 512 | **99.92%** | **3158** | 105 | 9.18ms | 12.12ms | 261s |
-| 512 | 32 | 512 | **99.95%** | **2956** | 99 | 9.8ms | 12.86ms | 354s |
-| 512 | 64 | 512 | **99.98%** | **2581** | 86 | 11.21ms | 15.68ms | 388s |
-
-
-
diff --git a/_includes/ann-sift-128.mdx b/_includes/ann-sift-128.mdx
deleted file mode 100644
index 89e74c43a7..0000000000
--- a/_includes/ann-sift-128.mdx
+++ /dev/null
@@ -1,75 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-
-
-
-|efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 64 | 8 | 64 | **90.91%** | **11445** | 381 | 2.59ms | 3.44ms | 186s |
-| 512 | 8 | 64 | **95.74%** | **11391** | 380 | 2.6ms | 3.4ms | 286s |
-| 128 | 16 | 64 | **98.52%** | **10443** | 348 | 2.83ms | 3.77ms | 204s |
-| 512 | 16 | 64 | **98.69%** | **10287** | 343 | 2.87ms | 3.94ms | 314s |
-| 128 | 32 | 64 | **98.92%** | **9760**| 325 | 3.03ms | 4.15ms | 203s |
-| 256 | 32 | 64 | **99.0%** | **9462** | 315 | 3.13ms | 4.36ms | 243s |
-| 512 | 32 | 64 | **99.22%** | **9249** | 308 | 3.2ms | 4.68ms | 351s |
-| 512 | 32 | 128 | **99.29%** | **7155** | 238 | 4.14ms | 5.84ms | 351s |
-| 128 | 32 | 256 | **99.34%** | **5694** | 190 | 5.21ms | 6.94ms | 203s |
-| 256 | 32 | 512 | **99.37%** | **3578** | 119 | 8.27ms | 11.2ms | 243s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 128 | 8 | 64 | **91.33%** | **16576** | 553 | 1.75ms | 2.82ms | 178s |
-| 256 | 8 | 64 | **91.98%** | **16474** | 549 | 1.76ms | 2.87ms | 205s |
-| 512 | 8 | 64 | **92.13%** | **16368** | 546 | 1.77ms | 2.85ms | 272s |
-| 64 | 16 | 64 | **96.56%** | **15003** | 500 | 1.93ms | 2.94ms | 160s |
-| 512 | 16 | 64 | **97.95%** | **14996** | 500 | 1.92ms | 2.78ms | 308s |
-| 64 | 64 | 64 | **98.04%** | **14197** | 473 | 2.05ms | 3.14ms | 167s |
-| 128 | 32 | 64 | **99.06%** | **13482** | 449 | 2.17ms | 3.07ms | 184s |
-| 256 | 32 | 64 | **99.44%** | **13237** | 441 | 2.2ms | 3.22ms | 261s |
-| 512 | 32 | 64 | **99.56%** | **12661** | 422 | 2.31ms | 3.32ms | 354s |
-| 256 | 64 | 64 | **99.63%** | **12014** | 400 | 2.43ms | 3.37ms | 276s |
-| 512 | 64 | 64 | **99.76%** | **11300** | 377 | 2.58ms | 3.56ms | 388s |
-| 512 | 32 | 128 | **99.76%** | **9365** | 312 | 3.14ms | 4.73ms | 354s |
-| 256 | 64 | 128 | **99.79%** | **8669** | 289 | 3.34ms | 4.67ms | 276s |
-| 512 | 64 | 128 | **99.89%** | **7990** | 266 | 3.65ms | 5.09ms | 388s |
-| 256 | 32 | 256 | **99.95%** | **6771** | 226 | 4.32ms | 5.84ms | 261s |
-| 512 | 32 | 256 | **99.97%** | **6286** | 210 | 4.66ms | 6.33ms | 354s |
-| 512 | 64 | 256 | **99.99%** | **5225** | 174 | 5.55ms | 8.11ms | 388s |
-| 256 | 32 | 512 | **100.0%** | **4281** | 143 | 6.84ms | 9.55ms | 261s |
-| 512 | 32 | 512 | **100.0%** | **3917** | 131 | 7.47ms | 10.33ms | 354s |
-| 256 | 64 | 512 | **100.0%** | **3611** | 120 | 8.03ms | 12.03ms | 276s |
-
-
-
-
-| efConstruction | maxConnections | ef | **Recall** | **QPS** | QPS/vCore | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 64 | 8 | 64 | **78.16%** | **6202** | 207 | 4.55ms | 6.21ms | 152s |
-| 256 | 8 | 64 | **80.07%** | **6044** | 201 | 4.59ms | 8.59ms | 205s |
-| 64 | 8 | 128 | **81.93%** | **5968** | 199 | 4.73ms | 6.98ms | 152s |
-| 512 | 16 | 64 | **91.28%** | **5930** | 198 | 4.75ms | 6.86ms | 308s |
-| 64 | 64 | 64 | **92.52%** | **5768** | 192 | 4.91ms | 6.38ms | 167s |
-| 128 | 16 | 128 | **93.17%** | **5650** | 188 | 5.02ms | 6.47ms | 185s |
-| 128 | 32 | 64 | **94.91%** | **5543** | 185 | 5.13ms | 6.81ms | 184s |
-| 256 | 32 | 64 | **96.07%** | **5524** | 184 | 5.12ms | 6.71ms | 261s |
-| 512 | 32 | 64 | **96.45%** | **5321** | 177 | 5.32ms | 7.51ms | 354s |
-| 128 | 32 | 128 | **96.54%** | **5254** | 175 | 5.42ms | 7.01ms | 184s |
-| 256 | 32 | 128 | **97.48%** | **5235** | 175 | 5.43ms | 7.34ms | 261s |
-| 512 | 32 | 128 | **97.79%** | **5045** | 168 | 5.65ms | 7.15ms | 354s |
-| 256 | 64 | 128 | **98.21%** | **4889** | 163 | 5.86ms | 7.75ms | 276s |
-| 512 | 64 | 128 | **98.75%** | **4667** | 156 | 6.13ms | 7.85ms | 388s |
-| 128 | 32 | 256 | **99.01%** | **4298** | 143 | 6.71ms | 8.76ms | 184s |
-| 256 | 32 | 256 | **99.43%** | **4242** | 141 | 6.77ms | 8.74ms | 261s |
-| 512 | 32 | 256 | **99.57%** | **4069** | 136 | 7.1ms | 9.01ms | 354s |
-| 256 | 64 | 256 | **99.61%** | **3854** | 128 | 7.47ms | 10.13ms | 276s |
-| 512 | 64 | 256 | **99.79%** | **3634** | 121 | 7.92ms | 10.88ms | 388s |
-| 256 | 32 | 512 | **99.92%** | **3158** | 105 | 9.18ms | 12.12ms | 261s |
-| 512 | 32 | 512 | **99.95%** | **2956** | 99 | 9.8ms | 12.86ms | 354s |
-| 512 | 64 | 512 | **99.98%** | **2581** | 86 | 11.21ms | 15.68ms | 388s |
-
-
-
diff --git a/_includes/clients/matrix-intro.md b/_includes/clients/matrix-intro.md
deleted file mode 100644
index 7c53efd701..0000000000
--- a/_includes/clients/matrix-intro.md
+++ /dev/null
@@ -1,53 +0,0 @@
-For links to the client releases associated with the latest Weaviate core releases, expand this section.
-
-
- Current core and client releases
-
-|Weaviate Version|Release Date|Python|TypeScript|Go|Java|
-|:-|:-|:-:|:-:|:-:|:-:|
-| [1.26.5](https://github.com/weaviate/weaviate/releases/tag/v1.26.5) | 2024-09-27 | 4.8.1 | 3.1.4 | 4.15.1 | 4.8.3 |
-| [1.26.4](https://github.com/weaviate/weaviate/releases/tag/v1.26.4) | 2024-09-06 | 4.7.1 | '' | '' | 4.8.2 |
-| [1.26.3](https://github.com/weaviate/weaviate/releases/tag/v1.26.3) | 2024-08-29 | '' | '' | '' | '' |
-| [1.26.2](https://github.com/weaviate/weaviate/releases/tag/v1.26.2) | 2024-08-29 | '' | '' | '' | '' |
-| [1.26.1](https://github.com/weaviate/weaviate/releases/tag/v1.26.1) | 2024-07-18 | 4.6.6 | 3.0.8 | 4.14.2 | 4.8.0 |
-| [1.26.0](https://github.com/weaviate/weaviate/releases/tag/v1.26.0) | 2024-07-18 | '' | '' | '' | 4.7.0 |
-
-
-
-For links to previous releases, expand this section.
-
-
- Previous core and client releases
-
-|Weaviate Version|Release Date|Python|TypeScript|Go|Java|
-|:-|:-|:-:|:-:|:-:|:-:|
-| [1.25.0](https://github.com/weaviate/weaviate/releases/tag/v1.25.0) | 2024-05-10 | 4.6.0 | 2.1.1 | 4.13.1 | 4.6.0 |
-| [1.24.0](https://github.com/weaviate/weaviate/releases/tag/v1.24.0) | 2024-02-27 | 4.5.0 | 2.1.0 | 4.12.0 | 4.5.1 |
-| [1.23.0](https://github.com/weaviate/weaviate/releases/tag/v1.23.0) | 2023-12-18 | 3.26.0 | 2.0.0 | '' |4.4.2 |
-| [1.22.0](https://github.com/weaviate/weaviate/releases/tag/v1.22.0) | 2023-10-27 | 3.25.0 | '' | 4.10.0 | 4.3.0 |
-| [1.21.0](https://github.com/weaviate/weaviate/releases/tag/v1.21.0) | 2023-08-17 | 3.22.1 |1.4.0 | 4.9.0 | 4.2.1 |
-| [1.20.0](https://github.com/weaviate/weaviate/releases/tag/v1.20.0) | 2023-07-06 | 3.22.0 | '' | '' | 4.2.0 |
-| [1.19.0](https://github.com/weaviate/weaviate/releases/tag/v1.19.0) | 2023-05-04 | 3.17.0 | 1.1.0] | 4.7.1 | 4.0.1 |
-| [1.18.0](https://github.com/weaviate/weaviate/releases/tag/v1.18.0) | 2023-03-07 | 3.13.0 | 2.14.5 | 4.6.2 | 3.6.4 |
-| [1.17.0](https://github.com/weaviate/weaviate/releases/tag/v1.17.0) | 2022-12-20 | 3.9.0 | 2.14.0 | 4.5.0 | 3.5.0 |
-| [1.16.0](https://github.com/weaviate/weaviate/releases/tag/v1.16.0) | 2022-10-31 | 3.8.0 | 2.13.0 | 4.4.0 | 3.4.0 |
-| [1.15.0](https://github.com/weaviate/weaviate/releases/tag/v1.15.0) | 2022-09-07 | '' | 2.12.0 | 4.3.0 | 3.3.0 |
-| [1.14.0](https://github.com/weaviate/weaviate/releases/tag/v1.14.0) | 2022-07-07 | 3.6.0 | 2.11.0 | 4.2.0 | 3.2.0 |
-| [1.13.0](https://github.com/weaviate/weaviate/releases/tag/v1.13.0) | 2022-05-03 | 3.4.2 | 2.9.0 | 4.0.0 | 2.4.0 |
-| [1.12.0](https://github.com/weaviate/weaviate/releases/tag/v1.12.0) | 2022-04-05 | 3.4.0 | 2.8.0 | 3.0.0 | '' |
-| [1.11.0](https://github.com/weaviate/weaviate/releases/tag/v1.11.0) | 2022-03-14 | 3.2.5 | 2.7.0 | 2.6.0 | 2.3.0 |
-| [1.10.0](https://github.com/weaviate/weaviate/releases/tag/v1.10.0) | 2022-01-27 | '' | 2.5.0 | 2.4.1 | 2.1.1 |
-| [1.9.0](https://github.com/weaviate/weaviate/releases/tag/v1.9.0) | 2021-12-10 | '' | '' | 2.4.0 | 2.1.0 |
-| [1.8.0](https://github.com/weaviate/weaviate/releases/tag/v1.8.0) | 2021-11-30 | '' | '' | '' | '' |
-| [1.7.0](https://github.com/weaviate/weaviate/releases/tag/v1.7.0) | 2021-09-01 | 3.1.1 | 2.4.0 | 2.3.0 | 1.1.0 |
-| [1.6.0](https://github.com/weaviate/weaviate/releases/tag/v1.6.0) | 2021-08-11 | 2.4.0 | 2.3.0 | 2.2.0 | '' |
-| [1.5.0](https://github.com/weaviate/weaviate/releases/tag/v1.5.0) | 2021-07-13 | '' | '' | '' | '' |
-| [1.4.0](https://github.com/weaviate/weaviate/releases/tag/v1.4.0) | 2021-06-09 | '' | '' | '' | '' |
-| [1.3.0](https://github.com/weaviate/weaviate/releases/tag/v1.3.0) | 2021-04-23 | '' | 2.1.0 | 2.1.0 | 1.0.0 |
-| [1.2.0](https://github.com/weaviate/weaviate/releases/tag/v1.2.0) | 2021-03-15 | 2.2.0 | 2.0.0 | 1.1.0 | - |
-| [1.1.0](https://github.com/weaviate/weaviate/releases/tag/v1.1.0) | 2021-02-10 | 2.1.0 | '' | '' | - |
-| [1.0.0](https://github.com/weaviate/weaviate/releases/tag/v1.0.0) | 2021-01-14 | 2.0.0 | '' | '' | - |
-
-Note: The TypeScript client replaced the JavaScript client on 2023-03-17.
-
-
diff --git a/_includes/code/client-libraries/migrate-ts3.ts b/_includes/code/client-libraries/migrate-ts3.ts
index 62a5269d0f..b03978b8ea 100644
--- a/_includes/code/client-libraries/migrate-ts3.ts
+++ b/_includes/code/client-libraries/migrate-ts3.ts
@@ -1,5 +1,5 @@
// CompleteScript // Imports
-import weaviate from 'weaviate-client';
+import weaviate, { WeaviateClient} from 'weaviate-client';
import 'dotenv/config';
// END CompleteScript // END Imports
@@ -11,15 +11,16 @@ async function main() {
// CompleteScript // CreateClient
// connect to your Weaviate instance on WCD
- const client = await weaviate.connectToWeaviateCloud(
- process.env.WEAVIATE_URL || '',
- {
- authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY || ''),
- headers: {
- 'X-OpenAI-Api-Key': process.env.OPENAI_API_KEY || '',
- }
+ const weaviateURL = process.env.WEAVIATE_URL as string
+ const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+ const openaiKey = process.env.OPENAI_API_KEY as string
+
+ const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
+ headers: {
+ 'X-OpenAI-Api-Key': openaiKey,
}
- )
+ })
// END CompleteScript // END CreateClient
// CompleteScript // CreateCollection
diff --git a/_includes/code/client-libraries/python_v4.py b/_includes/code/client-libraries/python_v4.py
index fcdbabdddb..bcee7158b1 100644
--- a/_includes/code/client-libraries/python_v4.py
+++ b/_includes/code/client-libraries/python_v4.py
@@ -1,3 +1,5 @@
+import weaviate
+
# START WVCImportExample
import weaviate.classes as wvc
# END WVCImportExample
@@ -5,15 +7,6 @@
import weaviate_datasets as wd
import os
-# InstantiationV3API
-import weaviate
-
-client = weaviate.Client(
- url="http://localhost:8080",
-)
-# END InstantiationV3API
-
-
# TryFinallyExample
import weaviate
diff --git a/_includes/code/configuration/replication-consistency.mdx b/_includes/code/configuration/replication-consistency.mdx
index 917c78c825..8209c4767a 100644
--- a/_includes/code/configuration/replication-consistency.mdx
+++ b/_includes/code/configuration/replication-consistency.mdx
@@ -22,7 +22,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
text={PyCodeV3}
startMarker="# START AsyncRepair"
endMarker="# END AsyncRepair"
- language="py"
+ language="pyv3"
/>
@@ -40,7 +40,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
text={TSCodeLegacy}
startMarker="// START AsyncRepair"
endMarker="// END AsyncRepair"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/connections/connect-python-v3.py b/_includes/code/connections/connect-python-v3.py
index ee16968786..62b8b5d2ed 100644
--- a/_includes/code/connections/connect-python-v3.py
+++ b/_includes/code/connections/connect-python-v3.py
@@ -1,5 +1,13 @@
# THIS FILE HASN'T BEEN TESTED TO RUN END-TO-END
+# InstantiationV3API
+import weaviate
+
+client = weaviate.Client(
+ url="http://localhost:8080",
+)
+# END InstantiationV3API
+
#####################
### Cloud connect ###
#####################
diff --git a/_includes/code/connections/connect-ts-v3.ts b/_includes/code/connections/connect-ts-v3.ts
index 9bf5a6fecb..dbe0825dbd 100644
--- a/_includes/code/connections/connect-ts-v3.ts
+++ b/_includes/code/connections/connect-ts-v3.ts
@@ -9,12 +9,14 @@
// WEAVIATE_URL your WCD instance URL
// WEAVIATE_API_KEY your WCD instance API key
+
import weaviate, { WeaviateClient } from 'weaviate-client';
-const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- process.env.WEAVIATE_URL,
- {
- authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY),
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
}
)
// END APIKeyWCD
@@ -30,10 +32,11 @@ const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
import weaviate, { WeaviateClient } from 'weaviate-client';
-const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- process.env.WEAVIATE_URL,
- {
- authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY),
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
timeout: { init: 30, query: 60, insert: 120 } // Values in seconds
}
)
@@ -63,8 +66,10 @@ console.log(client)
import weaviate from 'weaviate-client'
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+
const client = await weaviate.connectToLocal(
- { authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY),}
+ { authCredentials: new weaviate.ApiKey(weaviateKey),}
)
console.log(client)
@@ -80,10 +85,12 @@ console.log(client)
import weaviate from 'weaviate-client'
+const cohereKey = process.env.COHERE_API_KEY as string
+
const client = await weaviate.connectToLocal(
{
headers: {
- 'X-Cohere-Api-Key': process.env.COHERE_API_KEY || '',
+ 'X-Cohere-Api-Key': cohereKey,
}
}
)
@@ -120,12 +127,14 @@ console.log(client)
import weaviate, { WeaviateClient } from 'weaviate-client';
-const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- process.env.WEAVIATE_URL,
- {
- authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY),
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+const cohereKey = process.env.COHERE_API_KEY as string
+
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
headers: {
- 'X-Cohere-Api-Key': process.env.COHERE_API_KEY || '',
+ 'X-Cohere-Api-Key': cohereKey,
}
}
)
@@ -143,6 +152,7 @@ const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
import weaviate, { WeaviateClient } from 'weaviate-client';
+
const client = await weaviate.connectToCustom(
{
httpHost: process.env.WEAVIATE_URL, // URL only, no http prefix
diff --git a/_includes/code/connections/oidc-connect.mdx b/_includes/code/connections/oidc-connect.mdx
index e328836723..e0d64f3a08 100644
--- a/_includes/code/connections/oidc-connect.mdx
+++ b/_includes/code/connections/oidc-connect.mdx
@@ -23,7 +23,7 @@ import GoCode from '!!raw-loader!/_includes/code/connections/connect.go';
text={PyV3Code}
startMarker="# START OIDCConnect"
endMarker="# END OIDCConnect"
- language="py"
+ language="pyv3"
/>
@@ -39,7 +39,7 @@ import GoCode from '!!raw-loader!/_includes/code/connections/connect.go';
text={TsV2Code}
startMarker="// START OIDCConnect"
endMarker="// END OIDCConnect"
- language="js"
+ language="tsv2"
/>
diff --git a/_includes/code/core.client.palm.apikey.mdx b/_includes/code/core.client.palm.apikey.mdx
index c262941e75..f610352aee 100644
--- a/_includes/code/core.client.palm.apikey.mdx
+++ b/_includes/code/core.client.palm.apikey.mdx
@@ -11,8 +11,8 @@ client = weaviate.Client(
url = "https://WEAVIATE_INSTANCE_URL", # Replace WEAVIATE_INSTANCE_URL with the URL
# highlight-start
additional_headers = {
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY", # Replace with your API key
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY", # Replace with your API key
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY", # Replace with your API key
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY", # Replace with your API key
}
# highlight-end
)
@@ -30,8 +30,8 @@ const client = weaviate.client({
// highlight-start
// Replace with your API key
headers: {
- 'X-Google-Vertex-Api-Key': 'YOUR-VERTEX-API-KEY', // Replace with your API key
- 'X-Google-Studio-Api-Key': 'YOUR-AI-STUDIO-API-KEY', // Replace with your API key
+ 'X-Goog-Vertex-Api-Key': 'YOUR-VERTEX-API-KEY', // Replace with your API key
+ 'X-Goog-Studio-Api-Key': 'YOUR-AI-STUDIO-API-KEY', // Replace with your API key
},
// highlight-end
});
@@ -57,8 +57,8 @@ func main() {
// highlight-start
// Replace with your API key
Headers: map[string]string{
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY", // Replace with your API key
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY", // Replace with your API key
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY", // Replace with your API key
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY", // Replace with your API key
}
// highlight-end
}
@@ -85,8 +85,8 @@ public class App {
// highlight-start
Map headers = new HashMap() { {
// Replace with your API key
- put("X-Google-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
- put("X-Google-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
+ put("X-Goog-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
+ put("X-Goog-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
} };
// highlight-end
diff --git a/_includes/code/embedded.instantiate.custom.mdx b/_includes/code/embedded.instantiate.custom.mdx
index 87662112f9..c67f9cc1d4 100644
--- a/_includes/code/embedded.instantiate.custom.mdx
+++ b/_includes/code/embedded.instantiate.custom.mdx
@@ -20,7 +20,7 @@ import TsCodeV2CustomModules from '!!raw-loader!/_includes/code/install/embedded
text={PyCodeV3}
startMarker="# START CustomModules"
endMarker="# END CustomModules"
- language="py"
+ language="pyv3"
/>
@@ -28,7 +28,7 @@ import TsCodeV2CustomModules from '!!raw-loader!/_includes/code/install/embedded
text={TsCodeV2CustomModules}
startMarker="// START CustomModules"
endMarker="// END CustomModules"
- language="js"
+ language="tsv2"
/>
diff --git a/_includes/code/embedded.instantiate.mdx b/_includes/code/embedded.instantiate.mdx
index 526cfddcef..44c532495e 100644
--- a/_includes/code/embedded.instantiate.mdx
+++ b/_includes/code/embedded.instantiate.mdx
@@ -19,7 +19,7 @@ import TsCodeV2 from '!!raw-loader!/_includes/code/install/embedded-v2.ts';
text={PyCodeV3}
startMarker="# START SimpleInstance"
endMarker="# END SimpleInstance"
- language="py"
+ language="pyv3"
/>
@@ -27,7 +27,7 @@ import TsCodeV2 from '!!raw-loader!/_includes/code/install/embedded-v2.ts';
text={TsCodeV2}
startMarker="// START SimpleInstance"
endMarker="// END SimpleInstance"
- language="js"
+ language="tsv2"
/>
diff --git a/_includes/code/embedded.instantiate.module.mdx b/_includes/code/embedded.instantiate.module.mdx
index b9cb623510..ee7e307181 100644
--- a/_includes/code/embedded.instantiate.module.mdx
+++ b/_includes/code/embedded.instantiate.module.mdx
@@ -20,7 +20,7 @@ import TsCodeV2CustomModules from '!!raw-loader!/_includes/code/install/embedded
text={PyCodeV3}
startMarker="# START CustomModules"
endMarker="# END CustomModules"
- language="py"
+ language="pyv3"
/>
@@ -28,7 +28,7 @@ import TsCodeV2CustomModules from '!!raw-loader!/_includes/code/install/embedded
text={TsCodeV2CustomModules}
startMarker="// START CustomModules"
endMarker="// END CustomModules"
- language="js"
+ language="tsv2"
/>
diff --git a/_includes/code/generative.groupedtask.examples.py b/_includes/code/generative.groupedtask.examples.py
index 985347f7d1..7d7d314bad 100644
--- a/_includes/code/generative.groupedtask.examples.py
+++ b/_includes/code/generative.groupedtask.examples.py
@@ -16,8 +16,8 @@
"X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY"),
# END GenerativeOpenAI
# START GenerativeGoogle
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
# END GenerativeGoogle
# START GenerativeHuggingface
"X-HuggingFace-Api-Key": "YOUR_HUGGINGFACE_APIKEY",
diff --git a/_includes/code/generative.palm.groupedresult.mdx b/_includes/code/generative.palm.groupedresult.mdx
index 04f0daf21b..a318467fd2 100644
--- a/_includes/code/generative.palm.groupedresult.mdx
+++ b/_includes/code/generative.palm.groupedresult.mdx
@@ -20,8 +20,8 @@ import weaviate
client = weaviate.Client(
url = "https://WEAVIATE_INSTANCE_URL/", # Replace WEAVIATE_INSTANCE_URL with your instance URL
additional_headers={
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
}
)
@@ -97,8 +97,8 @@ func main() {
Host: "WEAVIATE_INSTANCE_URL", // Replace with your instance URL
Scheme: "https",
Headers: map[string]string{
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
},
}
client, err := weaviate.NewClient(cfg)
@@ -150,8 +150,8 @@ import io.weaviate.client.v1.graphql.query.fields.Field;
public class App {
public static void main(String[] args) {
Map headers = new HashMap() { {
- put("X-Google-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
- put("X-Google-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
+ put("X-Goog-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
+ put("X-Goog-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
} };
Config config = new Config("https", "WEAVIATE_INSTANCE_URL", headers);
// Replace with your instance URL
@@ -253,8 +253,8 @@ echo '{
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $WEAVIATE_API_KEY" \
- -H "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY" \
- -H "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY" \
+ -H "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY" \
+ -H "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY" \
-d @- \
https://WEAVIATE_INSTANCE_URL/v1/graphql # Replace WEAVIATE_INSTANCE_URL with your instance URL
```
diff --git a/_includes/code/generative.palm.singleresult.mdx b/_includes/code/generative.palm.singleresult.mdx
index c3a285359f..0cc08592a5 100644
--- a/_includes/code/generative.palm.singleresult.mdx
+++ b/_includes/code/generative.palm.singleresult.mdx
@@ -20,8 +20,8 @@ import weaviate
client = weaviate.Client(
url = "https://WEAVIATE_INSTANCE_URL", # Replace WEAVIATE_INSTANCE_URL with the URL
additional_headers={
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
}
)
@@ -96,8 +96,8 @@ func main() {
Host: "WEAVIATE_INSTANCE_URL",
Scheme: "https",
Headers: map[string]string{
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
},
}
client, err := weaviate.NewClient(cfg)
@@ -152,8 +152,8 @@ import io.weaviate.client.v1.graphql.query.fields.Field;
public class App {
public static void main(String[] args) {
Map headers = new HashMap() { {
- put("X-Google-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
- put("X-Google-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
+ put("X-Goog-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
+ put("X-Goog-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
} };
Config config = new Config("https", "WEAVIATE_INSTANCE_URL", headers);
WeaviateClient client = new WeaviateClient(config);
@@ -263,8 +263,8 @@ echo '{
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $WEAVIATE_API_KEY" \
- -H "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY" \
- -H "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY" \
+ -H "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY" \
+ -H "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY" \
-d @- \
https://${WEAVIATE_INSTANCE_URL}/v1/graphql
```
diff --git a/_includes/code/generative.singleprompt.examples.py b/_includes/code/generative.singleprompt.examples.py
index c4eb68f568..b443f818f5 100644
--- a/_includes/code/generative.singleprompt.examples.py
+++ b/_includes/code/generative.singleprompt.examples.py
@@ -16,8 +16,8 @@
"X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY"),
# END GenerativeOpenAI
# START GenerativeGoogle
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
# END GenerativeGoogle
# START GenerativeHuggingface
"X-HuggingFace-Api-Key": "YOUR_HUGGINGFACE_APIKEY",
diff --git a/_includes/code/graphql.filters.nearText.palm.mdx b/_includes/code/graphql.filters.nearText.palm.mdx
index 6a036463b3..aac309929a 100644
--- a/_includes/code/graphql.filters.nearText.palm.mdx
+++ b/_includes/code/graphql.filters.nearText.palm.mdx
@@ -20,8 +20,8 @@ import weaviate
client = weaviate.Client(
url="http://localhost:8080",
additional_headers={
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
}
)
@@ -100,8 +100,8 @@ func main() {
Host: "localhost:8080",
Scheme: "http",
Headers: map[string]string{
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
},
}
client, err := weaviate.NewClient(cfg)
@@ -170,8 +170,8 @@ import java.util.Map;
public class App {
public static void main(String[] args) {
Map headers = new HashMap() { {
- put("X-Google-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
- put("X-Google-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
+ put("X-Goog-Vertex-Api-Key", "YOUR-VERTEX-API-KEY");
+ put("X-Goog-Studio-Api-Key", "YOUR-AI-STUDIO-API-KEY");
} };
Config config = new Config("http", "localhost:8080", headers);
WeaviateClient client = new WeaviateClient(config);
@@ -249,8 +249,8 @@ echo '{
-X POST \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer learn-weaviate' \
- -H "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY" \
- -H "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY" \
+ -H "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY" \
+ -H "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY" \
-d @- \
https://edu-demo.weaviate.network/v1/graphql
```
diff --git a/_includes/code/graphql.filters.nearText.py b/_includes/code/graphql.filters.nearText.py
index cb00221c9e..d635e09d99 100644
--- a/_includes/code/graphql.filters.nearText.py
+++ b/_includes/code/graphql.filters.nearText.py
@@ -17,8 +17,8 @@
"X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY"),
# END NearTextOpenAI
# START NearTextGoogle
- "X-Google-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
- "X-Google-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
+ "X-Goog-Vertex-Api-Key": "YOUR-VERTEX-API-KEY",
+ "X-Goog-Studio-Api-Key": "YOUR-AI-STUDIO-API-KEY",
# END NearTextGoogle
# START NearTextHuggingface
"X-HuggingFace-Api-Key": "YOUR_HUGGINGFACE_APIKEY",
diff --git a/_includes/code/graphql.get.beacon.mdx b/_includes/code/graphql.get.beacon.mdx
index 928e461fc3..565fa45427 100644
--- a/_includes/code/graphql.get.beacon.mdx
+++ b/_includes/code/graphql.get.beacon.mdx
@@ -22,7 +22,7 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/graphql.get.beacon.v3.py';
text={PyCodeV3}
startMarker="# ===== PYTHON EXAMPLE ====="
endMarker="# ===== END PYTHON EXAMPLE ====="
- language="py"
+ language="pyv3"
/>
@@ -167,7 +167,7 @@ echo '{
text={PyCodeV3}
startMarker="# ===== GRAPHQL EXAMPLE ====="
endMarker="# ===== END GRAPHQL EXAMPLE ====="
- language="py"
+ language="pyv3"
/>
diff --git a/_includes/code/graphql.get.multitenancy.mdx b/_includes/code/graphql.get.multitenancy.mdx
index e3d31f535a..d1d1345441 100644
--- a/_includes/code/graphql.get.multitenancy.mdx
+++ b/_includes/code/graphql.get.multitenancy.mdx
@@ -22,7 +22,7 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.multi
text={PyCodeV3}
startMarker="# START Search"
endMarker="# END Search"
- language="py"
+ language="pyv3"
/>
@@ -31,7 +31,7 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.multi
text={TSCode}
startMarker="// START Search"
endMarker="// END Search"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/graphql.get.simple.mdx b/_includes/code/graphql.get.simple.mdx
index efe548a448..1a7c299339 100644
--- a/_includes/code/graphql.get.simple.mdx
+++ b/_includes/code/graphql.get.simple.mdx
@@ -20,7 +20,7 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/graphql.get.simple.v3.py';
text={PyCodeV3}
startMarker="# ===== PYTHON EXAMPLE ====="
endMarker="# ===== END PYTHON EXAMPLE ====="
- language="py"
+ language="pyv3"
/>
@@ -145,7 +145,7 @@ echo '{
text={PyCodeV3}
startMarker="# ===== GRAPHQL EXAMPLE ====="
endMarker="# ===== END GRAPHQL EXAMPLE ====="
- language="py"
+ language="pyv3"
/>
diff --git a/_includes/code/howto/configure.backups.py b/_includes/code/howto/configure.backups.py
index 50f1b0db4a..a2543ab6a0 100644
--- a/_includes/code/howto/configure.backups.py
+++ b/_includes/code/howto/configure.backups.py
@@ -92,9 +92,29 @@
# ===== Cancel ongoing backup =====
# ==============================================
+# Note - this will fail in automated tests as the backup is already completed
+
+# Create the collections, whether they exist or not
+client.collections.delete(["Article", "Publication"])
+articles = client.collections.create(name="Article")
+publications = client.collections.create(name="Publication")
+
+articles.data.insert(properties={"title": "Dummy"})
+publications.data.insert(properties={"title": "Dummy"})
+
+# Start a backup to cancel
+result = client.backup.create(
+ backup_id="some-unwanted-backup",
+ backend="filesystem",
+ include_collections=["Article", "Publication"],
+ wait_for_completion=False,
+)
+
+print(result)
+
# START CancelBackup
result = client.backup.cancel(
- backup_id="my-very-first-backup",
+ backup_id="some-unwanted-backup",
backend="filesystem",
)
diff --git a/_includes/code/howto/configure.backups.status.ts b/_includes/code/howto/configure.backups.status.ts
index 0cbf9a5c2d..7508a3fb1d 100644
--- a/_includes/code/howto/configure.backups.status.ts
+++ b/_includes/code/howto/configure.backups.status.ts
@@ -24,3 +24,12 @@ let restoreStatus = await client.backup.getRestoreStatus({
console.log(restoreStatus);
// END StatusRestoreBackup
+
+// START CancelBackup
+let cancelStatus = await client.backup.cancel({
+ backupId: 'my-very-first-backup',
+ backend: 'filesystem'
+})
+
+console.log(cancelStatus);
+// END CancelBackup
diff --git a/_includes/code/howto/go/docs/mainpkg/search-aggregation_test.go b/_includes/code/howto/go/docs/mainpkg/search-aggregation_test.go
index bb1e3954b5..49a8ca957e 100644
--- a/_includes/code/howto/go/docs/mainpkg/search-aggregation_test.go
+++ b/_includes/code/howto/go/docs/mainpkg/search-aggregation_test.go
@@ -11,7 +11,6 @@ import (
"github.com/weaviate/weaviate-go-client/v4/weaviate/graphql"
)
-
// ===============================
// ===== meta count EXAMPLES =====
// ===============================
@@ -42,7 +41,7 @@ func TestMetaCount(t *testing.T) {
assert.Contains(t, response.Data["Aggregate"], "JeopardyQuestion")
assert.Len(t, response.Data["Aggregate"].(map[string]interface{})["JeopardyQuestion"].([]interface{}), 1)
assert.Contains(t, response.Data["Aggregate"].(map[string]interface{})["JeopardyQuestion"].([]interface{})[0], "meta")
-
+
meta := response.Data["Aggregate"].(map[string]interface{})["JeopardyQuestion"].([]interface{})[0].(map[string]interface{})["meta"].(map[string]interface{})
assert.Equal(t, float64(216930), meta["count"]) //or 10,000 if you are using the smaller dataset
}
@@ -101,7 +100,7 @@ func TestIntProp(t *testing.T) {
Fields: []graphql.Field{
{Name: "count"},
{Name: "sum"},
- // highlight-end
+ // highlight-end
},
}).
Do(ctx)
@@ -112,7 +111,7 @@ func TestIntProp(t *testing.T) {
t.Logf("%s", jsonResponse)
points := response.Data["Aggregate"].(map[string]interface{})["JeopardyQuestion"].([]interface{})[0].(map[string]interface{})["points"].(map[string]interface{})
- assert.Equal(t, float64(216930), points["count"]) //or 10,000 if you are using the smaller dataset
+ assert.Equal(t, float64(216930), points["count"]) //or 10,000 if you are using the smaller dataset
assert.Equal(t, float64(141871083), points["sum"]) // or 6324100 if you are using the smaller dataset
}
@@ -209,7 +208,7 @@ func TestNearTextWithDistance(t *testing.T) {
WithConcepts([]string{"animals in space"}).
// highlight-start
WithDistance(0.19)).
- // highlight-end
+ // highlight-end
WithFields(graphql.Field{
Name: "points",
Fields: []graphql.Field{
@@ -227,6 +226,40 @@ func TestNearTextWithDistance(t *testing.T) {
assert.Greater(t, points["sum"].(float64), float64(0))
}
+// =========================================
+// ===== Hybrid EXAMPLES =====
+// =========================================
+
+func TestHybridAggregate(t *testing.T) {
+ client := setupClient()
+ ctx := context.Background()
+
+ // HybridExample
+ // Go client support coming soon
+ // END HybridExample
+ response, err := client.GraphQL().Aggregate().
+ WithClassName("JeopardyQuestion").
+ WithNearText(client.GraphQL().NearTextArgBuilder().
+ WithConcepts([]string{"animals in space"})).
+ // highlight-start
+ WithObjectLimit(10).
+ // highlight-end
+ WithFields(graphql.Field{
+ Name: "points",
+ Fields: []graphql.Field{
+ {Name: "sum"},
+ },
+ }).
+ Do(ctx)
+
+ require.NoError(t, err)
+ jsonResponse, _ := json.MarshalIndent(response, "", " ")
+ t.Logf("%s", jsonResponse)
+
+ points := response.Data["Aggregate"].(map[string]interface{})["JeopardyQuestion"].([]interface{})[0].(map[string]interface{})["points"].(map[string]interface{})
+ assert.Greater(t, points["sum"].(float64), float64(0))
+}
+
// =================================
// ===== where filter EXAMPLES =====
// =================================
@@ -236,8 +269,8 @@ func TestWhereFilter(t *testing.T) {
ctx := context.Background()
// START whereFilter
- // Add this line to imports: "github.com/weaviate/weaviate-go-client/v4/weaviate/filters"
-
+ // Add this line to imports: "github.com/weaviate/weaviate-go-client/v4/weaviate/filters"
+
response, err := client.GraphQL().Aggregate().
WithClassName("JeopardyQuestion").
// highlight-start
diff --git a/_includes/code/howto/go/docs/model-providers/1-connect-weaviate-embeddings/main.go b/_includes/code/howto/go/docs/model-providers/1-connect-weaviate-embeddings/main.go
new file mode 100644
index 0000000000..9c8aadcb28
--- /dev/null
+++ b/_includes/code/howto/go/docs/model-providers/1-connect-weaviate-embeddings/main.go
@@ -0,0 +1,43 @@
+// START-ANY
+// Set these environment variables
+// WCD_HOSTNAME your Weaviate instance hostname
+// WCD_API_KEY your Weaviate instance API key
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+ "github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
+)
+
+func main() {
+ cfg := weaviate.Config{
+ Host: os.Getenv("WCD_W_EMB_HOSTNAME"),
+ Scheme: "https",
+ AuthConfig: auth.ApiKey{Value: os.Getenv("WCD_W_EMB_API_KEY")},
+ Headers: map[string]string{
+ "X-Weaviate-Api-Key": os.Getenv("WCD_W_EMB_API_KEY"),
+ "X-Weaviate-Cluster-Url": fmt.Sprintf("https://%s", os.Getenv("WCD_W_EMB_HOSTNAME")),
+ },
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // Work with Weaviate
+ // END-ANY
+ isReady, err := client.Misc().ReadyChecker().Do(context.Background())
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("%v", isReady)
+ // START-ANY
+}
+
+// END-ANY
diff --git a/_includes/code/howto/go/docs/model-providers/1-connect/main.go b/_includes/code/howto/go/docs/model-providers/1-connect/main.go
new file mode 100644
index 0000000000..9ffea82a9c
--- /dev/null
+++ b/_includes/code/howto/go/docs/model-providers/1-connect/main.go
@@ -0,0 +1,78 @@
+// START-ANY
+// Set these environment variables
+// WCD_HOSTNAME your Weaviate instance hostname, excluding the scheme (https://)
+// WCD_API_KEY your Weaviate instance API key
+// _APIKEY your model provider API key (or token)
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+ "github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
+)
+
+func main() {
+ cfg := weaviate.Config{
+ Host: os.Getenv("WCD_HOSTNAME"),
+ Scheme: "https",
+ AuthConfig: auth.ApiKey{Value: os.Getenv("WCD_API_KEY")},
+ // highlight-start
+ Headers: map[string]string{
+ // END-ANY
+ // START AWSInstantiation
+ "X-AWS-Access-Key": os.Getenv("AWS_ACCESS_KEY"),
+ "X-AWS-Secret-Key": os.Getenv("AWS_SECRET_KEY"),
+ // END AWSInstantiation
+ // START CohereInstantiation
+ "X-Cohere-Api-Key": os.Getenv("COHERE_APIKEY"),
+ // END CohereInstantiation
+ // START DatabricksInstantiation
+ "X-Databricks-Token": os.Getenv("DATABRICKS_TOKEN"),
+ // END DatabricksInstantiation
+ // START GoogleInstantiation
+ "X-Goog-Vertex-Key": os.Getenv("VERTEX_APIKEY"),
+ "X-Goog-Studio-Key": os.Getenv("STUDIO_APIKEY"),
+ // END GoogleInstantiation
+ // START HuggingFaceInstantiation
+ "X-HuggingFace-Api-Key": os.Getenv("HUGGINGFACE_APIKEY"),
+ // END HuggingFaceInstantiation
+ // START JinaAIInstantiation
+ "X-JinaAI-Api-Key": os.Getenv("JINAAI_APIKEY"),
+ // END JinaAIInstantiation
+ // START MistralInstantiation
+ "X-Mistral-Api-Key": os.Getenv("MISTRAL_APIKEY"),
+ // END MistralInstantiation
+ // START OpenAIInstantiation
+ "X-OpenAI-Api-Key": os.Getenv("OPENAI_APIKEY"),
+ // END OpenAIInstantiation
+ // START AzureOpenAIInstantiation
+ "X-Azure-Api-Key": os.Getenv("AZURE_APIKEY"),
+ // END AzureOpenAIInstantiation
+ // START VoyageAIInstantiation
+ "X-VoyageAI-Api-Key": os.Getenv("VOYAGEAI_APIKEY"),
+ // END VoyageAIInstantiation
+ // START-ANY
+ },
+ // highlight-end
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // Work with Weaviate
+ // END-ANY
+ isReady, err := client.Misc().ReadyChecker().Do(context.Background())
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("%v", isReady)
+ // START-ANY
+}
+
+// END-ANY
diff --git a/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go b/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go
new file mode 100644
index 0000000000..e44b327cff
--- /dev/null
+++ b/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go
@@ -0,0 +1,1087 @@
+// Set these environment variables
+// WCD_HOSTNAME your Weaviate instance hostname
+// WCD_API_KEY your Weaviate instance API key
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "os"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+ "github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
+ "github.com/weaviate/weaviate-go-client/v4/weaviate/fault"
+ "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql"
+ "github.com/weaviate/weaviate/entities/models"
+)
+
+// START-ANY
+// package, imports not shown
+
+func main() {
+ // Instantiation not shown
+
+ ctx := context.Background()
+
+ // END-ANY
+ cfg := weaviate.Config{
+ Host: os.Getenv("WCD_HOSTNAME"),
+ Scheme: "https",
+ AuthConfig: auth.ApiKey{Value: os.Getenv("WCD_API_KEY")},
+ Headers: map[string]string{
+ "X-Cohere-Api-Key": os.Getenv("COHERE_APIKEY"),
+ },
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerAWSBedrock
+ // highlight-start
+ // Define the collection
+ basicAWSBedrockVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-aws": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "region": "us-east-1",
+ "service": "bedrock",
+ "model": "cohere.embed-multilingual-v3",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicAWSBedrockVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerAWSBedrock
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerAWSSagemaker
+ // highlight-start
+ // Define the collection
+ basicAWSSagemakerVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-aws": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "region": "us-east-1",
+ "service": "sagemaker",
+ "endpoint": "",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicAWSSagemakerVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerAWSSagemaker
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerAWS
+ // highlight-start
+ // Define the collection
+ awsVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-aws": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "region": "us-east-1",
+ "service": "bedrock", // "bedrock" or "sagemaker"
+ "model": "cohere.embed-multilingual-v3", // If using `bedrock`, this is required
+ // "endpoint": "", // If using `sagemaker`, this is required
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(awsVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerAWS
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerCohere
+ // highlight-start
+ // Define the collection
+ basicCohereVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-cohere": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicCohereVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerCohere
+
+ // START VectorizerCohereCustomModel
+ // highlight-start
+ // Define the collection
+ cohereVectorizerWithModelDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-cohere": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "embed-multilingual-v3.0",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(cohereVectorizerWithModelDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END VectorizerCohereCustomModel
+
+ // START FullVectorizerCohere
+ // highlight-start
+ // Define the collection
+ cohereVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-cohere": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ // "model": "embed-multilingual-v3.0",
+ // "truncate": "END", // "NONE", "START" or "END"
+ // "base_url": "",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(cohereVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerCohere
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerDatabricks
+ // highlight-start
+ // Define the collection
+ basicDatabricksVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-databricks": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "endpoint": "", // Required for Databricks
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicDatabricksVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerDatabricks
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerGoogleVertex
+ // highlight-start
+ // Define the collection
+ basicGoogleVertexVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-google": map[string]interface{}{
+ "project_id": "",
+ "model_id": "textembedding-gecko@latest", // (Optional) To manually set the model ID
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicGoogleVertexVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerGoogleVertex
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerGoogleStudio
+ // highlight-start
+ // Define the collection
+ basicGoogleStudioVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-google": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model_id": "text-embedding-004", // (Optional) To manually set the model ID
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicGoogleStudioVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerGoogleStudio
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerGoogle
+ // highlight-start
+ // Define the collection
+ googleVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-aws": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "project_id": "", // Required for Vertex AU
+ "model_id": "textembedding-gecko@latest", // (Optional) To manually set the model ID
+ "api_endpoint": "", // (Optional) To manually set the API endpoint
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(googleVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerGoogle
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerHuggingFace
+ // highlight-start
+ // Define the collection
+ basicHuggingfaceVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-huggingface": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "sentence-transformers/all-MiniLM-L6-v2",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicHuggingfaceVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerHuggingFace
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerHuggingFace
+ // highlight-start
+ // Define the collection
+ fullHuggingfaceVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-huggingface": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ // Note: Use only one of (`model`), (`passage_model` and `query_model`), or (`endpoint_url`)
+ "model": "sentence-transformers/all-MiniLM-L6-v2",
+ // "passage_model": "sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", // Required if using `query_model`
+ // "query_model": "sentence-transformers/facebook-dpr-question_encoder-single-nq-base", // Required if using `passage_model`
+ // "endpoint_url": "",
+ // // Optional parameters
+ // "wait_for_model": true,
+ // "use_cache": true,
+ // "use_gpu": true,
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(fullHuggingfaceVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerHuggingFace
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerJinaAI
+ // highlight-start
+ // Define the collection
+ basicJinaVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-jinaai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicJinaVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerJinaAI
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START VectorizerJinaCustomModel
+ // highlight-start
+ // Define the collection
+ jinaVectorizerWithModelDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-jinaai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "jina-embeddings-v3",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(jinaVectorizerWithModelDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END VectorizerJinaCustomModel
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerJinaAI
+ // highlight-start
+ // Define the collection
+ jinaVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-jinaai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "jina-embeddings-v3",
+ "dimensions": 512, // e.g. 1024, 512, 256 (only applicable for some models)
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(jinaVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerJinaAI
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerMistral
+ // highlight-start
+ // Define the collection
+ basicMistralVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-mistral": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicMistralVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerMistral
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerMistral
+ // highlight-start
+ // Define the collection
+ mistralVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-mistral": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "mistral-embed",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(mistralVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerMistral
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerOpenAI
+ // highlight-start
+ // Define the collection
+ basicOpenAIVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-openai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicOpenAIVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerOpenAI
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START VectorizerOpenAICustomModelV3
+ // highlight-start
+ // Define the collection
+ openAIVectorizerWithModelDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-openai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "text-embedding-3-large",
+ "dimensions": 1024, // Optional (e.g. 1024, 512, 256)
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(openAIVectorizerWithModelDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END VectorizerOpenAICustomModelV3
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START VectorizerOpenAICustomModelLegacy
+ // highlight-start
+ // Define the collection
+ openAIVectorizerWithLegacyModelDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-openai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "ada",
+ "model_version": "002",
+ "type": "text",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(openAIVectorizerWithLegacyModelDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END VectorizerOpenAICustomModelLegacy
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerOpenAI
+ // highlight-start
+ // Define the collection
+ openAIVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-openai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "text-embedding-3-large",
+ "dimensions": 1024, // Parameter only applicable for `v3` model family and newer
+ "model_version": "002", // Parameter only applicable for `ada` model family and older
+ "type": "text", // Parameter only applicable for `ada` model family and older
+ "base_url": "",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(openAIVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerOpenAI
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerAzureOpenAI
+ // highlight-start
+ // Define the collection
+ basicAzureVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-openai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "resourceName": "",
+ "deploymentID": "",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicAzureVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerAzureOpenAI
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerAzureOpenAI
+ // highlight-start
+ // Define the collection
+ azureVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-openai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "resourceName": "",
+ "deploymentID": "",
+ // "baseURL": "",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(azureVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerAzureOpenAI
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START BasicVectorizerVoyageAI
+ // highlight-start
+ // Define the collection
+ basicVoyageVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-voyageai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicVoyageVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerVoyageAI
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START VectorizerVoyageAICustomModel
+ // highlight-start
+ // Define the collection
+ voyageVectorizerWithModelDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-voyageai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ "model": "voyage-3-lite",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(voyageVectorizerWithModelDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END VectorizerVoyageAICustomModel
+
+ // Clean slate: Delete the collection
+ if err := client.Schema().ClassDeleter().WithClassName("DemoCollection").Do(context.Background()); err != nil {
+ // Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+ if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+ panic(err)
+ }
+ }
+
+ // START FullVectorizerVoyageAI
+ // highlight-start
+ // Define the collection
+ voyageVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-voyageai": map[string]interface{}{
+ "sourceProperties": []string{"title"},
+ // "model": "voyage-3-lite",
+ // "base_url": "",
+ // "truncate": true,
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(voyageVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerVoyageAI
+
+ // START BasicVectorizerWeaviate
+ // highlight-start
+ // Define the collection
+ basicWeaviateVectorizerDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-weaviate": map[string]interface{}{},
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(basicWeaviateVectorizerDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END BasicVectorizerWeaviate
+
+ // START VectorizerWeaviateCustomModel
+ // highlight-start
+ // Define the collection
+ weaviateVectorizerWithModelDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-weaviate": map[string]interface{}{
+ "model": "arctic-embed-m-v1.5",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(weaviateVectorizerWithModelDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END VectorizerWeaviateCustomModel
+
+ // START FullVectorizerWeaviate
+ // highlight-start
+ // Define the collection
+ weaviateVectorizerFullDef := &models.Class{
+ Class: "DemoCollection",
+ VectorConfig: map[string]models.VectorConfig{
+ "title_vector": {
+ Vectorizer: map[string]interface{}{
+ "text2vec-weaviate": map[string]interface{}{
+ "model": "arctic-embed-m-v1.5",
+ "dimensions": 256, // Or 756
+ "base_url": "",
+ },
+ },
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(weaviateVectorizerFullDef).Do(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+ // END FullVectorizerWeaviate
+
+ // START BatchImportExample
+ var sourceObjects = []map[string]string{
+ // Objects not shown
+ // END BatchImportExample
+ {"title": "The Shawshank Redemption", "description": "Prison drama about hope"},
+ {"title": "The Godfather", "description": "Mafia family epic"},
+ {"title": "The Dark Knight", "description": "Batman vs Joker thriller"},
+ {"title": "Jingle All the Way", "description": "Holiday shopping adventure"},
+ {"title": "A Christmas Carol", "description": "Ghost story of redemption"},
+ // START BatchImportExample
+ }
+
+ // highlight-start
+ // Convert items into a slice of models.Object
+ objects := []models.PropertySchema{}
+ for i := range sourceObjects {
+ objects = append(objects, map[string]interface{}{
+ // Populate the object with the data
+ // END BatchImportExample
+ "title": sourceObjects[i]["title"],
+ "description": sourceObjects[i]["description"],
+ // START BatchImportExample
+ })
+ }
+
+ // Batch write items
+ batcher := client.Batch().ObjectsBatcher()
+ for _, dataObj := range objects {
+ batcher.WithObjects(&models.Object{
+ Class: "DemoCollection",
+ Properties: dataObj,
+ })
+ }
+
+ // Flush
+ batchRes, err := batcher.Do(ctx)
+
+ // Error handling
+ if err != nil {
+ panic(err)
+ }
+ for _, res := range batchRes {
+ if res.Result.Errors != nil {
+ for _, err := range res.Result.Errors.Error {
+ if err != nil {
+ fmt.Printf("Error details: %v\n", *err)
+ panic(err.Message)
+ }
+ }
+ }
+ }
+ // highlight-end
+ // END BatchImportExample
+
+ // START NearTextExample
+ // highlight-start
+ nearTextResponse, err := client.GraphQL().Get().
+ WithClassName("DemoCollection").
+ WithFields(
+ graphql.Field{Name: "title"},
+ ).
+ WithNearText(client.GraphQL().NearTextArgBuilder().
+ WithConcepts([]string{"A holiday film"})).
+ WithLimit(2).
+ Do(ctx)
+ // highlight-end
+
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("%v", nearTextResponse)
+ // END NearTextExample
+
+ // START HybridExample
+ // highlight-start
+ hybridResponse, err := client.GraphQL().Get().
+ WithClassName("DemoCollection").
+ WithFields(
+ graphql.Field{Name: "title"},
+ ).
+ WithHybrid(client.GraphQL().HybridArgumentBuilder().
+ WithQuery("A holiday film")).
+ WithLimit(2).
+ Do(ctx)
+ // highlight-end
+
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("%v", hybridResponse)
+ // END HybridExample
+
+ // START-ANY
+}
+
+// END-ANY
diff --git a/_includes/code/howto/go/docs/quickstart/1_is_ready/main.go b/_includes/code/howto/go/docs/quickstart/1_is_ready/main.go
index 92f81c7cc1..17a66b8701 100644
--- a/_includes/code/howto/go/docs/quickstart/1_is_ready/main.go
+++ b/_includes/code/howto/go/docs/quickstart/1_is_ready/main.go
@@ -15,6 +15,7 @@ import (
)
func main() {
+ // highlight-start
cfg := weaviate.Config{
Host: os.Getenv("WCD_HOSTNAME"),
Scheme: "https",
@@ -22,6 +23,7 @@ func main() {
}
client, err := weaviate.NewClient(cfg)
+ // highlight-end
if err != nil {
fmt.Println(err)
}
diff --git a/_includes/code/howto/go/docs/quickstart/2_1_create_collection/main.go b/_includes/code/howto/go/docs/quickstart/2_1_create_collection/main.go
index e7802f46a7..a7e50e17ab 100644
--- a/_includes/code/howto/go/docs/quickstart/2_1_create_collection/main.go
+++ b/_includes/code/howto/go/docs/quickstart/2_1_create_collection/main.go
@@ -2,7 +2,7 @@
// Set these environment variables
// WCD_HOSTNAME your Weaviate instance hostname
// WCD_API_KEY your Weaviate instance API key
-// OPENAI_API_KEY your OpenAI API key
+// COHERE_APIKEY your Cohere API key
package main
@@ -32,10 +32,10 @@ func main() {
// Define the collection
classObj := &models.Class{
Class: "Question",
- Vectorizer: "text2vec-openai",
+ Vectorizer: "text2vec-cohere",
ModuleConfig: map[string]interface{}{
- "text2vec-openai": map[string]interface{}{},
- "generative-openai": map[string]interface{}{},
+ "text2vec-cohere": map[string]interface{}{},
+ "generative-cohere": map[string]interface{}{},
},
}
diff --git a/_includes/code/howto/go/docs/quickstart/2_2_add_objects/main.go b/_includes/code/howto/go/docs/quickstart/2_2_add_objects/main.go
index 625b33a0f4..c410fc87e0 100644
--- a/_includes/code/howto/go/docs/quickstart/2_2_add_objects/main.go
+++ b/_includes/code/howto/go/docs/quickstart/2_2_add_objects/main.go
@@ -2,7 +2,7 @@
// Set these environment variables
// WCD_HOSTNAME your Weaviate instance hostname
// WCD_API_KEY your Weaviate instance API key
-// OPENAI_API_KEY your OpenAI API key
+// COHERE_APIKEY your Cohere API key
package main
@@ -25,7 +25,7 @@ func main() {
AuthConfig: auth.ApiKey{Value: os.Getenv("WCD_API_KEY")},
// highlight-start
Headers: map[string]string{
- "X-OpenAI-Api-Key": os.Getenv("OPENAI_API_KEY"),
+ "X-Cohere-Api-Key": os.Getenv("COHERE_APIKEY"),
},
// highlight-end
}
diff --git a/_includes/code/howto/go/docs/quickstart/3_1_neartext/main.go b/_includes/code/howto/go/docs/quickstart/3_1_neartext/main.go
index 7cc9e62d7d..8bee24389d 100644
--- a/_includes/code/howto/go/docs/quickstart/3_1_neartext/main.go
+++ b/_includes/code/howto/go/docs/quickstart/3_1_neartext/main.go
@@ -2,7 +2,7 @@
// Set these environment variables
// WCD_HOSTNAME your Weaviate instance hostname
// WCD_API_KEY your Weaviate instance API key
-// OPENAI_API_KEY your OpenAI API key
+// COHERE_APIKEY your Cohere API key
package main
@@ -23,7 +23,7 @@ func main() {
AuthConfig: auth.ApiKey{Value: os.Getenv("WCD_API_KEY")},
// highlight-start
Headers: map[string]string{
- "X-OpenAI-Api-Key": os.Getenv("OPENAI_API_KEY"),
+ "X-Cohere-Api-Key": os.Getenv("COHERE_APIKEY"),
},
// highlight-end
}
diff --git a/_includes/code/howto/go/docs/quickstart/3_2_rag/main.go b/_includes/code/howto/go/docs/quickstart/3_2_rag/main.go
index ccd73bd07a..5846172c05 100644
--- a/_includes/code/howto/go/docs/quickstart/3_2_rag/main.go
+++ b/_includes/code/howto/go/docs/quickstart/3_2_rag/main.go
@@ -2,7 +2,7 @@
// Set these environment variables
// WCD_HOSTNAME your Weaviate instance hostname
// WCD_API_KEY your Weaviate instance API key
-// OPENAI_API_KEY your OpenAI API key
+// COHERE_APIKEY your Cohere API key
package main
@@ -23,7 +23,7 @@ func main() {
AuthConfig: auth.ApiKey{Value: os.Getenv("WCD_API_KEY")},
// highlight-start
Headers: map[string]string{
- "X-OpenAI-Api-Key": os.Getenv("OPENAI_API_KEY"),
+ "X-Cohere-Api-Key": os.Getenv("COHERE_APIKEY"),
},
// highlight-end
}
diff --git a/_includes/code/howto/go/docs/quickstart_local/1_is_ready/main.go b/_includes/code/howto/go/docs/quickstart_local/1_is_ready/main.go
new file mode 100644
index 0000000000..c6dee0df9d
--- /dev/null
+++ b/_includes/code/howto/go/docs/quickstart_local/1_is_ready/main.go
@@ -0,0 +1,32 @@
+// START InstantiationExample
+package main
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+)
+
+func main() {
+ // highlight-start
+ cfg := weaviate.Config{
+ Host: "localhost:8080",
+ Scheme: "http",
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ // highlight-end
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // Check the connection
+ ready, err := client.Misc().ReadyChecker().Do(context.Background())
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("%v", ready)
+}
+
+// END InstantiationExample
diff --git a/_includes/code/howto/go/docs/quickstart_local/2_1_create_collection/main.go b/_includes/code/howto/go/docs/quickstart_local/2_1_create_collection/main.go
new file mode 100644
index 0000000000..bf8f064c7e
--- /dev/null
+++ b/_includes/code/howto/go/docs/quickstart_local/2_1_create_collection/main.go
@@ -0,0 +1,48 @@
+// START CreateCollection
+package main
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+ "github.com/weaviate/weaviate/entities/models"
+)
+
+func main() {
+ cfg := weaviate.Config{
+ Host: "localhost:8080",
+ Scheme: "http",
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // highlight-start
+ // Define the collection
+ classObj := &models.Class{
+ Class: "Question",
+ Vectorizer: "text2vec-ollama",
+ ModuleConfig: map[string]interface{}{
+ "text2vec-ollama": map[string]interface{}{ // Configure the Ollama embedding integration
+ "apiEndpoint": "http://host.docker.internal:11434", // Allow Weaviate from within a Docker container to contact your Ollama instance
+ "model": "nomic-embed-text", // The model to use
+ },
+ "generative-ollama": map[string]interface{}{ // Configure the Ollama generative integration
+ "apiEndpoint": "http://host.docker.internal:11434", // Allow Weaviate from within a Docker container to contact your Ollama instance
+ "model": "llama3.2", // The model to use
+ },
+ },
+ }
+
+ // add the collection
+ err = client.Schema().ClassCreator().WithClass(classObj).Do(context.Background())
+ if err != nil {
+ panic(err)
+ }
+ // highlight-end
+}
+
+// END CreateCollection
diff --git a/_includes/code/howto/go/docs/quickstart_local/2_2_add_objects/main.go b/_includes/code/howto/go/docs/quickstart_local/2_2_add_objects/main.go
new file mode 100644
index 0000000000..6f3510605e
--- /dev/null
+++ b/_includes/code/howto/go/docs/quickstart_local/2_2_add_objects/main.go
@@ -0,0 +1,65 @@
+// Import
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+ "github.com/weaviate/weaviate/entities/models"
+)
+
+func main() {
+ cfg := weaviate.Config{
+ Host: "localhost:8080",
+ Scheme: "http",
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // Retrieve the data
+ data, err := http.DefaultClient.Get("https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/jeopardy_tiny.json")
+ if err != nil {
+ panic(err)
+ }
+ defer data.Body.Close()
+
+ // Decode the data
+ var items []map[string]string
+ if err := json.NewDecoder(data.Body).Decode(&items); err != nil {
+ panic(err)
+ }
+
+ // highlight-start
+ // convert items into a slice of models.Object
+ objects := make([]*models.Object, len(items))
+ for i := range items {
+ objects[i] = &models.Object{
+ Class: "Question",
+ Properties: map[string]any{
+ "category": items[i]["Category"],
+ "question": items[i]["Question"],
+ "answer": items[i]["Answer"],
+ },
+ }
+ }
+
+ // batch write items
+ batchRes, err := client.Batch().ObjectsBatcher().WithObjects(objects...).Do(context.Background())
+ if err != nil {
+ panic(err)
+ }
+ for _, res := range batchRes {
+ if res.Result.Errors != nil {
+ panic(res.Result.Errors.Error)
+ }
+ }
+ // highlight-end
+}
+
+// END Import
diff --git a/_includes/code/howto/go/docs/quickstart_local/3_1_neartext/main.go b/_includes/code/howto/go/docs/quickstart_local/3_1_neartext/main.go
new file mode 100644
index 0000000000..fdd18d0809
--- /dev/null
+++ b/_includes/code/howto/go/docs/quickstart_local/3_1_neartext/main.go
@@ -0,0 +1,45 @@
+// NearText
+package main
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+ "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql"
+)
+
+func main() {
+ cfg := weaviate.Config{
+ Host: "localhost:8080",
+ Scheme: "http",
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // highlight-start
+ ctx := context.Background()
+
+ response, err := client.GraphQL().Get().
+ WithClassName("Question").
+ WithFields(
+ graphql.Field{Name: "question"},
+ graphql.Field{Name: "answer"},
+ graphql.Field{Name: "category"},
+ ).
+ WithNearText(client.GraphQL().NearTextArgBuilder().
+ WithConcepts([]string{"biology"})).
+ WithLimit(2).
+ Do(ctx)
+ // highlight-end
+
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("%v", response)
+}
+
+// END NearText
diff --git a/_includes/code/howto/go/docs/quickstart_local/3_2_rag/main.go b/_includes/code/howto/go/docs/quickstart_local/3_2_rag/main.go
new file mode 100644
index 0000000000..6bf7aee1f9
--- /dev/null
+++ b/_includes/code/howto/go/docs/quickstart_local/3_2_rag/main.go
@@ -0,0 +1,50 @@
+// RAG
+package main
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/weaviate/weaviate-go-client/v4/weaviate"
+ "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql"
+)
+
+func main() {
+ cfg := weaviate.Config{
+ Host: "localhost:8080",
+ Scheme: "http",
+ }
+
+ client, err := weaviate.NewClient(cfg)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ // highlight-start
+ ctx := context.Background()
+
+ generatePrompt := "Write a tweet with emojis about these facts."
+
+ gs := graphql.NewGenerativeSearch().GroupedResult(generatePrompt)
+
+ response, err := client.GraphQL().Get().
+ WithClassName("Question").
+ WithFields(
+ graphql.Field{Name: "question"},
+ graphql.Field{Name: "answer"},
+ graphql.Field{Name: "category"},
+ ).
+ WithGenerativeSearch(gs).
+ WithNearText(client.GraphQL().NearTextArgBuilder().
+ WithConcepts([]string{"biology"})).
+ WithLimit(2).
+ Do(ctx)
+ // highlight-end
+
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("%v", response)
+}
+
+// END RAG
diff --git a/_includes/code/howto/java/pom.xml b/_includes/code/howto/java/pom.xml
index e6f6ffee93..a58f5dd6d0 100644
--- a/_includes/code/howto/java/pom.xml
+++ b/_includes/code/howto/java/pom.xml
@@ -37,6 +37,11 @@
${weaviate.client.version}test
+
+ org.json
+ json
+ 20240303
+ org.junit.jupiterjunit-jupiter-api
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/manage-data.classes.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/manage-data.classes.java
index 4a436d9284..f478ec33f3 100644
--- a/_includes/code/howto/java/src/test/java/io/weaviate/docs/manage-data.classes.java
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/manage-data.classes.java
@@ -171,7 +171,7 @@ private void createCollectionWithNamedVectors(String className) {
Map text2vecCohere = new HashMap<>();
Map text2vecCohereSettings = new HashMap<>();
text2vecCohereSettings.put("properties", new String[]{ "body" });
- text2vecCohere.put("text2vec_cohere", text2vecCohereSettings);
+ text2vecCohere.put("text2vec-cohere", text2vecCohereSettings);
//Define the vector configurations
Map vectorConfig = new HashMap<>();
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/Connect.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/Connect.java
new file mode 100644
index 0000000000..41de8e4815
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/Connect.java
@@ -0,0 +1,51 @@
+package io.weaviate.docs.model_providers;
+
+// START-ANY
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+
+// END-ANY
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+// START-ANY
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+// _APIKEY Your model provider API key
+
+public class Connect {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+ String cohereKey = System.getenv("COHERE_APIKEY");
+
+ // highlight-start
+ Map headers = new HashMap() { {
+ // END-ANY
+ // START CohereInstantiation
+ put("X-Cohere-Api-Key", cohereKey);
+ // END CohereInstantiation
+ // START-ANY
+ } };
+
+ Config config = new Config("https", host, headers);
+ // highlight-end
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ // check the result
+ Result result = client.misc().readyChecker().run();
+ System.out.println(result.getResult());
+ // END-ANY
+
+ assertThat(result.getResult()).isTrue();
+ // START-ANY
+ }
+}
+// END-ANY
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ConnectWeaviateEmbeddings.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ConnectWeaviateEmbeddings.java
new file mode 100644
index 0000000000..e6f5721266
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ConnectWeaviateEmbeddings.java
@@ -0,0 +1,43 @@
+package io.weaviate.docs.model_providers;
+
+// START-ANY
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+// START-ANY
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+
+public class ConnectWeaviateEmbeddings {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_W_EMB_HOSTNAME");
+ String apiKey = System.getenv("WCD_W_EMB_API_KEY");
+
+ // highlight-start
+ Map headers = new HashMap() { {
+ put("X-Weaviate-Api-Key", apiKey);
+ put("X-Weaviate-Cluster-Url", "https://" + System.getenv("WCD_W_EMB_HOSTNAME"));
+ } };
+
+ // highlight-start
+ Config config = new Config("https", host, headers);
+ // highlight-end
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ // check the result
+ Result result = client.misc().readyChecker().run();
+ System.out.println(result.getResult());
+
+ // END-ANY
+ assertThat(result.getResult()).isTrue();
+ // START-ANY
+ }
+}
+// END-ANY
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ImportAndQueries.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ImportAndQueries.java
new file mode 100644
index 0000000000..b36d8d2125
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ImportAndQueries.java
@@ -0,0 +1,140 @@
+package io.weaviate.docs.model_providers;
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.batch.api.ObjectsBatcher;
+import io.weaviate.client.v1.data.model.WeaviateObject;
+import io.weaviate.client.v1.graphql.model.GraphQLResponse;
+import io.weaviate.client.v1.graphql.query.argument.HybridArgument;
+import io.weaviate.client.v1.graphql.query.argument.NearTextArgument;
+import io.weaviate.client.v1.graphql.query.builder.GetBuilder;
+import io.weaviate.client.v1.graphql.query.fields.Field;
+import io.weaviate.client.v1.graphql.query.fields.Fields;
+
+import java.util.*;
+
+public class ImportAndQueries {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+ String cohereKey = System.getenv("COHERE_APIKEY");
+
+ Map headers = new HashMap() {
+ {
+ put("X-Cohere-Api-Key", cohereKey);
+ }
+ };
+
+ Config config = new Config("https", host, headers);
+
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ // Define source objects
+ List> sourceObjects = Arrays.asList(
+ new HashMap() {{
+ put("title", "The Shawshank Redemption");
+ put("description", "");
+ }},
+ new HashMap() {{
+ put("title", "The Godfather");
+ put("description", "");
+ }},
+ new HashMap() {{
+ put("title", "The Dark Knight");
+ put("description", "");
+ }},
+ new HashMap() {{
+ put("title", "Jingle All the Way");
+ put("description", "");
+ }},
+ new HashMap() {{
+ put("title", "A Christmas Carol");
+ put("description", "");
+ }}
+ );
+
+ // START BatchImportExample
+ List> objects = new ArrayList<>();
+ for (Map sourceObject : sourceObjects) {
+ HashMap schema = new HashMap<>();
+ schema.put("title", sourceObject.get("title"));
+ schema.put("description", sourceObject.get("description"));
+ objects.add(schema);
+ }
+
+ // Batch write items
+ ObjectsBatcher batcher = client.batch().objectsBatcher();
+ for (Map properties : objects) {
+ batcher.withObject(WeaviateObject.builder()
+ .className("DemoCollection")
+ .properties(properties)
+ // .tenant("tenantA") // If multi-tenancy is enabled, specify the tenant to which the object will be added.
+ .build()
+ );
+ }
+
+ // Flush
+ batcher.run();
+ // END BatchImportExample
+
+ // START NearTextExample // START HybridExample
+ // highlight-start
+ Fields returnFields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("title").build(),
+ })
+ .build();
+
+ // highlight-end
+ // START NearTextExample // END HybridExample
+ // highlight-start
+ NearTextArgument nearText = NearTextArgument.builder()
+ .concepts(new String[]{"A holiday film"})
+ .build();
+
+ String nearTextQuery = GetBuilder.builder()
+ .className("DemoCollection")
+ .fields(returnFields)
+ .withNearTextFilter(nearText)
+ .limit(2)
+ .build()
+ .buildQuery();
+ // highlight-end
+
+ Result nearTextResult = client.graphQL().raw().withQuery(nearTextQuery).run();
+
+ if (nearTextResult.hasErrors()) {
+ System.err.println(nearTextResult.getError());
+ } else {
+ System.out.println("Near Text Results: " + nearTextResult.getResult().getData());
+ }
+ // END NearTextExample
+
+ // START HybridExample
+ // highlight-start
+ HybridArgument hybrid = HybridArgument.builder()
+ .query("A holiday film")
+ .build();
+
+ String hybridQuery = GetBuilder.builder()
+ .className("DemoCollection")
+ .fields(returnFields)
+ .withHybridFilter(hybrid)
+ .limit(2)
+ .build()
+ .buildQuery();
+ // highlight-end
+
+ Result hybridResult = client.graphQL().raw().withQuery(hybridQuery).run();
+
+ if (hybridResult.hasErrors()) {
+ System.err.println(hybridResult.getError());
+ } else {
+ System.out.println("Hybrid Results: " + hybridResult.getResult().getData());
+ }
+ // END HybridExample
+ }
+}
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageCohereTextEmbeddings.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageCohereTextEmbeddings.java
new file mode 100644
index 0000000000..2c6a630966
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageCohereTextEmbeddings.java
@@ -0,0 +1,65 @@
+package io.weaviate.docs.model_providers;
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.schema.model.WeaviateClass;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+// _APIKEY Your Provider API key
+
+public class UsageCohereTextEmbeddings {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+ String cohereKey = System.getenv("COHERE_APIKEY");
+
+ Map headers = new HashMap() {
+ {
+ put("X-Cohere-Api-Key", cohereKey);
+ }
+ };
+
+ Config config = new Config("https", host, headers);
+
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ client.schema().classDeleter().withClassName("DemoCollection").run();
+
+ // START BasicVectorizerCohere // START VectorizerCohereCustomModel // START FullVectorizerCohere
+ Map text2vecCohere = new HashMap<>();
+ Map text2vecCohereSettings = new HashMap<>();
+
+ text2vecCohereSettings.put("properties", new String[]{"title"});
+ // END BasicVectorizerCohere // START VectorizerCohereCustomModel // START FullVectorizerCohere
+ text2vecCohereSettings.put("model", new String[]{"embed-multilingual-light-v3.0"});
+ // END BasicVectorizerCohere // END VectorizerCohereCustomModel // START FullVectorizerCohere
+ // text2vecCohereSettings.put("truncate", new String[]{"END"}); // "NONE", "START" or "END"
+ // text2vecCohereSettings.put("base_url", new String[]{""});
+ // START BasicVectorizerCohere // START VectorizerCohereCustomModel // START FullVectorizerCohere
+ text2vecCohere.put("text2vec-cohere", text2vecCohereSettings);
+
+ // Define the vector configurations
+ Map vectorConfig = new HashMap<>();
+ vectorConfig.put("title_vector", WeaviateClass.VectorConfig.builder()
+ .vectorIndexType("hnsw")
+ .vectorizer(text2vecCohere)
+ .build());
+
+ // Create the collection "DemoCollection"
+ WeaviateClass clazz = WeaviateClass.builder()
+ .className("DemoCollection")
+ .vectorConfig(vectorConfig)
+ .build();
+
+ Result result = client.schema().classCreator().withClass(clazz).run();
+ // END BasicVectorizerCohere // END VectorizerCohereCustomModel // END FullVectorizerCohere
+ }
+}
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageWeaviateTextEmbeddings.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageWeaviateTextEmbeddings.java
new file mode 100644
index 0000000000..4906884cbe
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageWeaviateTextEmbeddings.java
@@ -0,0 +1,58 @@
+package io.weaviate.docs.model_providers;
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.schema.model.WeaviateClass;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+// _APIKEY Your Provider API key
+
+public class UsageWeaviateTextEmbeddings {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+
+ Config config = new Config("https", host);
+
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ client.schema().classDeleter().withClassName("DemoCollection").run();
+
+ // START BasicVectorizerWeaviate // START VectorizerWeaviateCustomModel // START FullVectorizerWeaviate
+ Map text2vecWeaviate = new HashMap<>();
+ Map text2vecWeaviateSettings = new HashMap<>();
+
+ text2vecWeaviateSettings.put("properties", new String[]{"title"});
+ // END BasicVectorizerWeaviate // START VectorizerWeaviateCustomModel // START FullVectorizerWeaviate
+ text2vecWeaviateSettings.put("model", new String[]{"arctic-embed-m-v1.5"});
+ // END BasicVectorizerWeaviate // END VectorizerWeaviateCustomModel // START FullVectorizerWeaviate
+ text2vecWeaviateSettings.put("dimensions", new Integer[]{768}); // 768, 256
+ text2vecWeaviateSettings.put("base_url", new String[]{""});
+ // START BasicVectorizerWeaviate // START VectorizerWeaviateCustomModel // START FullVectorizerWeaviate
+ text2vecWeaviate.put("text2vec-weaviate", text2vecWeaviateSettings);
+
+ // Define the vector configurations
+ Map vectorConfig = new HashMap<>();
+ vectorConfig.put("title_vector", WeaviateClass.VectorConfig.builder()
+ .vectorIndexType("hnsw")
+ .vectorizer(text2vecWeaviate)
+ .build());
+
+ // Create the collection "DemoCollection"
+ WeaviateClass clazz = WeaviateClass.builder()
+ .className("DemoCollection")
+ .vectorConfig(vectorConfig)
+ .build();
+
+ Result result = client.schema().classCreator().withClass(clazz).run();
+ // END BasicVectorizerWeaviate // END VectorizerWeaviateCustomModel // END FullVectorizerWeaviate
+ }
+}
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/CreateCollection.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/CreateCollection.java
new file mode 100644
index 0000000000..1019f42cac
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/CreateCollection.java
@@ -0,0 +1,54 @@
+package io.weaviate.docs.quickstart;
+
+// START CreateCollection
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.schema.model.WeaviateClass;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+
+public class CreateCollection {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+
+ Config config = new Config("https", host);
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ // END CreateCollection
+ client.schema().classDeleter().withClassName("Question").run();
+
+ // START CreateCollection
+ // highlight-start
+ Map text2vecCohereSettings = new HashMap<>();
+ Map generativeCohereSettings = new HashMap<>();
+
+ Map moduleConfig = new HashMap<>();
+ moduleConfig.put("text2vec-cohere", text2vecCohereSettings);
+ moduleConfig.put("generative-cohere", generativeCohereSettings);
+
+ // Create the collection "Question"
+ WeaviateClass clazz = WeaviateClass.builder()
+ .className("Question")
+ .vectorizer("text2vec-cohere")
+ .moduleConfig(moduleConfig)
+ .build();
+ // highlight-end
+
+ // Review the response
+ Result result = client.schema().classCreator().withClass(clazz).run();
+ System.out.println(result);
+ Result collectionDefinition = client.schema().classGetter().withClassName("Question").run();
+ System.out.println(collectionDefinition.getResult());
+ }
+}
+// END CreateCollection
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/Import.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/Import.java
new file mode 100644
index 0000000000..62c5108702
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/Import.java
@@ -0,0 +1,69 @@
+package io.weaviate.docs.quickstart;
+
+// START Import
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.v1.batch.api.ObjectsBatcher;
+import io.weaviate.client.v1.data.model.WeaviateObject;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+// COHERE_APIKEY Your Cohere API key
+
+public class Import {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+ String cohereKey = System.getenv("COHERE_APIKEY");
+
+ // highlight-start
+ Map headers = new HashMap() { {
+ put("X-Cohere-Api-Key", cohereKey);
+ } };
+
+ Config config = new Config("https", host, headers);
+ // highlight-end
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ // Get JSON data
+ URL url = new URL("https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/jeopardy_tiny.json");
+ String jsonData = new BufferedReader(new InputStreamReader(((HttpURLConnection) url.openConnection()).getInputStream()))
+ .lines().reduce("", String::concat);
+
+ // highlight-start
+ // Create and execute batch
+ ObjectsBatcher batcher = client.batch().objectsBatcher();
+
+ new JSONArray(jsonData).forEach(item -> {
+ JSONObject json = (JSONObject) item;
+ HashMap properties = new HashMap<>();
+ properties.put("category", json.getString("Category"));
+ properties.put("question", json.getString("Question"));
+ properties.put("answer", json.getString("Answer"));
+
+ batcher.withObject(WeaviateObject.builder()
+ .className("Question")
+ .properties(properties)
+ .build());
+ });
+
+ // Flush
+ batcher.run();
+ // highlight-end
+ }
+}
+// END Import
+
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/IsReady.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/IsReady.java
new file mode 100644
index 0000000000..adcd3dafa7
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/IsReady.java
@@ -0,0 +1,36 @@
+package io.weaviate.docs.quickstart;
+
+// START InstantiationExample
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.base.Result;
+
+// END InstantiationExample
+import static org.assertj.core.api.Assertions.assertThat;
+
+// START InstantiationExample
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+
+public class IsReady {
+ public static void main(String[] args) throws Exception {
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+
+ // highlight-start
+ Config config = new Config("https", host);
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+ // highlight-end
+
+ // check the result
+ Result result = client.misc().readyChecker().run();
+ System.out.println(result.getResult());
+
+ // END InstantiationExample
+ assertThat(result.getResult()).isTrue();
+ // START InstantiationExample
+ }
+}
+// END InstantiationExample
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/NearText.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/NearText.java
new file mode 100644
index 0000000000..c587dc874e
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/NearText.java
@@ -0,0 +1,63 @@
+package io.weaviate.docs.quickstart;
+
+// START NearText
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.graphql.model.GraphQLResponse;
+import io.weaviate.client.v1.graphql.query.argument.NearTextArgument;
+import io.weaviate.client.v1.graphql.query.builder.GetBuilder;
+import io.weaviate.client.v1.graphql.query.fields.Field;
+import io.weaviate.client.v1.graphql.query.fields.Fields;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+// COHERE_APIKEY Your Cohere API key
+
+public class NearText {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+ String cohereKey = System.getenv("COHERE_APIKEY");
+
+ // highlight-start
+ Map headers = new HashMap() { {
+ put("X-Cohere-Api-Key", cohereKey);
+ } };
+
+ Config config = new Config("https", host, headers);
+ // highlight-end
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ // highlight-start
+ NearTextArgument nearText = NearTextArgument.builder()
+ .concepts(new String[]{"biology"})
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("Question")
+ .fields(fields)
+ .withNearTextFilter(nearText)
+ .limit(2)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // highlight-end
+ System.out.println(result.getResult());
+ }
+}
+// END NearText
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/RAG.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/RAG.java
new file mode 100644
index 0000000000..fc97656ba4
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/RAG.java
@@ -0,0 +1,70 @@
+package io.weaviate.docs.quickstart;
+
+// START RAG
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateAuthClient;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.graphql.model.GraphQLResponse;
+import io.weaviate.client.v1.graphql.query.argument.NearTextArgument;
+import io.weaviate.client.v1.graphql.query.builder.GetBuilder;
+import io.weaviate.client.v1.graphql.query.fields.Field;
+import io.weaviate.client.v1.graphql.query.fields.Fields;
+import io.weaviate.client.v1.graphql.query.fields.GenerativeSearchBuilder;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// Set these environment variables
+// WCD_HOSTNAME Your Weaviate instance hostname
+// WCD_API_KEY Your Weaviate instance API key
+// COHERE_APIKEY Your Cohere API key
+
+public class RAG {
+ public static void main(String[] args) throws Exception {
+
+ String host = System.getenv("WCD_HOSTNAME");
+ String apiKey = System.getenv("WCD_API_KEY");
+ String cohereKey = System.getenv("COHERE_APIKEY");
+
+ // highlight-start
+ Map headers = new HashMap() { {
+ put("X-Cohere-Api-Key", cohereKey);
+ } };
+
+ Config config = new Config("https", host, headers);
+ // highlight-end
+ WeaviateClient client = WeaviateAuthClient.apiKey(config, apiKey);
+
+ // highlight-start
+ NearTextArgument nearText = NearTextArgument.builder()
+ .concepts(new String[]{"biology"})
+ .build();
+
+ GenerativeSearchBuilder ragQuery = GenerativeSearchBuilder.builder()
+ .groupedResultTask("Write a tweet with emojis about these facts.")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("Question")
+ .fields(fields)
+ .withNearTextFilter(nearText)
+ .withGenerativeSearch(ragQuery)
+ .limit(2)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // highlight-end
+ System.out.println(result.getResult());
+ }
+}
+// END RAG
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/CreateCollection.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/CreateCollection.java
new file mode 100644
index 0000000000..8df1c68a9a
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/CreateCollection.java
@@ -0,0 +1,50 @@
+package io.weaviate.docs.quickstart_local;
+
+// START CreateCollection
+import java.util.HashMap;
+import java.util.Map;
+
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.schema.model.WeaviateClass;
+
+public class CreateCollection {
+ public static void main(String[] args) {
+ Config config = new Config("http", "localhost:8080"); // Replace with your Weaviate endpoint
+
+ WeaviateClient client = new WeaviateClient(config);
+
+ // END CreateCollection
+ client.schema().classDeleter().withClassName("Question").run();
+
+ // START CreateCollection
+ // highlight-start
+ Map text2vecOllamaSettings = new HashMap<>();
+ text2vecOllamaSettings.put("apiEndpoint", "http://host.docker.internal:11434"); // Allow Weaviate from within a Docker container to contact your Ollama instance
+ text2vecOllamaSettings.put("model", "nomic-embed-text"); // The model to use
+
+ Map generativeOllamaSettings = new HashMap<>();
+ generativeOllamaSettings.put("apiEndpoint", "http://host.docker.internal:11434"); // Allow Weaviate from within a Docker container to contact your Ollama instance
+ generativeOllamaSettings.put("model", "llama3.2"); // The model to use
+
+ Map> moduleConfig = new HashMap<>();
+ moduleConfig.put("text2vec-ollama", text2vecOllamaSettings);
+ moduleConfig.put("generative-ollama", generativeOllamaSettings);
+
+ // Create the collection "Question"
+ WeaviateClass clazz = WeaviateClass.builder()
+ .className("Question")
+ .vectorizer("text2vec-ollama")
+ .moduleConfig(moduleConfig)
+ .build();
+ // highlight-end
+
+ // Review the response
+ Result result = client.schema().classCreator().withClass(clazz).run();
+ System.out.println(result);
+ Result collectionDefinition = client.schema().classGetter().withClassName("Question").run();
+ System.out.println(collectionDefinition.getResult());
+ }
+}
+// END CreateCollection
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/Import.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/Import.java
new file mode 100644
index 0000000000..083e63eec8
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/Import.java
@@ -0,0 +1,51 @@
+package io.weaviate.docs.quickstart_local;
+
+// START Import
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.v1.data.model.WeaviateObject;
+import io.weaviate.client.v1.batch.api.ObjectsBatcher;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.HashMap;
+
+public class Import {
+ public static void main(String[] args) throws Exception {
+ Config config = new Config("http", "localhost:8080"); // Replace with your Weaviate endpoint
+
+ WeaviateClient client = new WeaviateClient(config);
+
+ // Get JSON data
+ URL url = new URL("https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/jeopardy_tiny.json");
+ String jsonData = new BufferedReader(new InputStreamReader(((HttpURLConnection) url.openConnection()).getInputStream()))
+ .lines().reduce("", String::concat);
+
+ // highlight-start
+ // Create and execute batch
+ ObjectsBatcher batcher = client.batch().objectsBatcher();
+
+ new JSONArray(jsonData).forEach(item -> {
+ JSONObject json = (JSONObject) item;
+ HashMap properties = new HashMap<>();
+ properties.put("category", json.getString("Category"));
+ properties.put("question", json.getString("Question"));
+ properties.put("answer", json.getString("Answer"));
+
+ batcher.withObject(WeaviateObject.builder()
+ .className("Question")
+ .properties(properties)
+ .build());
+ });
+
+ // Flush
+ batcher.run();
+ // highlight-end
+ }
+}
+// END Import
+
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/IsReady.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/IsReady.java
new file mode 100644
index 0000000000..97bc9be64d
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/IsReady.java
@@ -0,0 +1,28 @@
+package io.weaviate.docs.quickstart_local;
+
+// START InstantiationExample
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+
+// END InstantiationExample
+import static org.assertj.core.api.Assertions.assertThat;
+
+// START InstantiationExample
+public class IsReady {
+ public static void main(String[] args) {
+ // highlight-start
+ Config config = new Config("http", "localhost:8080");
+ WeaviateClient client = new WeaviateClient(config);
+ // highlight-end
+
+ // check the result
+ Result result = client.misc().readyChecker().run();
+ System.out.println(result.getResult());
+
+ // END InstantiationExample
+ assertThat(result.getResult()).isTrue();
+ // START InstantiationExample
+ }
+}
+// END InstantiationExample
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/NearText.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/NearText.java
new file mode 100644
index 0000000000..69b42d0a5d
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/NearText.java
@@ -0,0 +1,44 @@
+package io.weaviate.docs.quickstart_local;
+
+// START NearText
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.graphql.model.GraphQLResponse;
+import io.weaviate.client.v1.graphql.query.argument.NearTextArgument;
+import io.weaviate.client.v1.graphql.query.builder.GetBuilder;
+import io.weaviate.client.v1.graphql.query.fields.Field;
+import io.weaviate.client.v1.graphql.query.fields.Fields;
+
+public class NearText {
+ public static void main(String[] args) {
+ Config config = new Config("http", "localhost:8080"); // Replace with your Weaviate endpoint
+
+ WeaviateClient client = new WeaviateClient(config);
+
+ // highlight-start
+ NearTextArgument nearText = NearTextArgument.builder()
+ .concepts(new String[]{"biology"})
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("Question")
+ .fields(fields)
+ .withNearTextFilter(nearText)
+ .limit(2)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // highlight-end
+ System.out.println(result.getResult());
+ }
+}
+// END NearText
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/RAG.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/RAG.java
new file mode 100644
index 0000000000..cb039fc944
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/RAG.java
@@ -0,0 +1,50 @@
+package io.weaviate.docs.quickstart_local;
+
+// START RAG
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.graphql.model.GraphQLResponse;
+import io.weaviate.client.v1.graphql.query.argument.NearTextArgument;;
+import io.weaviate.client.v1.graphql.query.builder.GetBuilder;
+import io.weaviate.client.v1.graphql.query.fields.GenerativeSearchBuilder;
+import io.weaviate.client.v1.graphql.query.fields.Field;
+import io.weaviate.client.v1.graphql.query.fields.Fields;
+
+public class RAG {
+ public static void main(String[] args) {
+ Config config = new Config("http", "localhost:8080"); // Replace with your Weaviate endpoint
+
+ WeaviateClient client = new WeaviateClient(config);
+
+ // highlight-start
+ NearTextArgument nearText = NearTextArgument.builder()
+ .concepts(new String[]{"biology"})
+ .build();
+
+ GenerativeSearchBuilder ragQuery = GenerativeSearchBuilder.builder()
+ .groupedResultTask("Write a tweet with emojis about these facts.")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("Question")
+ .fields(fields)
+ .withNearTextFilter(nearText)
+ .withGenerativeSearch(ragQuery)
+ .limit(2)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // highlight-end
+ System.out.println(result.getResult());
+ }
+}
+// END RAG
diff --git a/_includes/code/howto/java/src/test/java/io/weaviate/docs/search/KeywordSearchTest.java b/_includes/code/howto/java/src/test/java/io/weaviate/docs/search/KeywordSearchTest.java
new file mode 100644
index 0000000000..4ae47c1082
--- /dev/null
+++ b/_includes/code/howto/java/src/test/java/io/weaviate/docs/search/KeywordSearchTest.java
@@ -0,0 +1,374 @@
+package io.weaviate.docs.search;
+
+import com.google.gson.GsonBuilder;
+import io.weaviate.client.Config;
+import io.weaviate.client.WeaviateClient;
+import io.weaviate.client.base.Result;
+import io.weaviate.client.v1.filters.Operator;
+import io.weaviate.client.v1.filters.WhereFilter;
+import io.weaviate.client.v1.graphql.model.GraphQLResponse;
+import io.weaviate.client.v1.graphql.query.argument.Bm25Argument;
+import io.weaviate.client.v1.graphql.query.argument.GroupByArgument;
+import io.weaviate.client.v1.graphql.query.argument.WhereArgument;
+import io.weaviate.client.v1.graphql.query.builder.GetBuilder;
+import io.weaviate.client.v1.graphql.query.fields.Field;
+import io.weaviate.client.v1.graphql.query.fields.Fields;
+import io.weaviate.client.v1.schema.model.DataType;
+import io.weaviate.client.v1.schema.model.Property;
+import io.weaviate.client.v1.schema.model.Tokenization;
+import io.weaviate.client.v1.schema.model.WeaviateClass;
+import io.weaviate.docs.helper.EnvHelper;
+import java.util.Arrays;
+import static org.assertj.core.api.Assertions.assertThat;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag("crud")
+@Tag("keyword-search")
+public class KeywordSearchTest {
+
+ private static WeaviateClient client;
+
+ @BeforeAll
+ public static void beforeAll() {
+ String scheme = EnvHelper.scheme("http");
+ String host = EnvHelper.host("localhost");
+ String port = EnvHelper.port("8080");
+
+ Config config = new Config(scheme, host + ":" + port);
+ client = new WeaviateClient(config);
+
+ Result result = client.schema().allDeleter().run();
+ assertThat(result).isNotNull()
+ .withFailMessage(() -> result.getError().toString())
+ .returns(false, Result::hasErrors)
+ .withFailMessage(null)
+ .returns(true, Result::getResult);
+ }
+
+ @Test
+ public void shouldPerformBasicSearch() {
+ String className = "JeopardyQuestion";
+ searchWithBM25(className);
+ searchWithBM25Score(className);
+ searchWithBM25OnSelectedProperties(className);
+ searchWithBM25WithWeights(className);
+ setTokenization(className);
+ searchWithBM25LimitOffset(className);
+ searchWithBM25LimitResultGroups(className);
+ searchWithBM25GroupResult(className);
+ searchWithBM25Filter(className);
+ }
+
+ private void searchWithBM25(String className) {
+ // START Basic
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("food")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .limit(3)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END Basic
+
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+ }
+
+ private void searchWithBM25Score(String className) {
+ // START Score
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("food")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ Field.builder().name("_additional").fields(new Field[]{
+ Field.builder().name("score").build()
+ }).build()
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .limit(3)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END Score
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+ }
+
+ private void searchWithBM25OnSelectedProperties(String className) {
+ // START Properties
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("food")
+ .properties(new String[]{ "question" })
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ Field.builder().name("_additional").fields(new Field[]{
+ Field.builder().name("score").build()
+ }).build()
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .limit(3)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END Properties
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+ }
+
+ private void searchWithBM25WithWeights(String className) {
+ // START Boost
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("food")
+ .properties(new String[]{ "question^2", "answer" })
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ Field.builder().name("_additional").fields(new Field[]{
+ Field.builder().name("score").build()
+ }).build()
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .limit(3)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END Boost
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+ }
+
+ private void setTokenization(String className) {
+ // START PropModuleSettings
+ Property titleProperty = Property.builder()
+ .name("title")
+ .description("title of the article")
+ .dataType(Arrays.asList(DataType.TEXT))
+ .tokenization(Tokenization.LOWERCASE)
+ .build();
+
+ Property bodyProperty = Property.builder()
+ .name("body")
+ .description("body of the article")
+ .dataType(Arrays.asList(DataType.TEXT))
+ .tokenization(Tokenization.WHITESPACE)
+ .build();
+
+ //Add the defined properties to the class
+ WeaviateClass articleClass = WeaviateClass.builder()
+ .className("Article")
+ .description("Article Class Description...")
+ .properties(Arrays.asList(titleProperty, bodyProperty))
+ .build();
+
+ Result result = client.schema().classCreator()
+ .withClass(articleClass)
+ .run();
+ // END PropModuleSettings
+ }
+
+ private void searchWithBM25LimitOffset(String className) {
+ // START limit
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("safety")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .limit(3)
+ .offset(1)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END limit
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+ }
+
+ private void searchWithBM25LimitResultGroups(String className) {
+ // START autocut
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("safety")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ })
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .autocut(1)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END autocut
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+ }
+
+ private void searchWithBM25GroupResult(String className) {
+ // START BM25GroupBy
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("California")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("round").build()
+ })
+ .build();
+
+ GroupByArgument groupBy = GroupByArgument.builder()
+ .path(new String[]{ "round" })
+ .groups(1)
+ .objectsPerGroup(3)
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .withGroupByArgument(groupBy)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END BM25GroupBy
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+ }
+
+ private void searchWithBM25Filter(String className) {
+ // START Filter
+ Bm25Argument keyword = Bm25Argument.builder()
+ .query("food")
+ .build();
+
+ Fields fields = Fields.builder()
+ .fields(new Field[]{
+ Field.builder().name("question").build(),
+ Field.builder().name("answer").build(),
+ Field.builder().name("round").build(),
+ Field.builder().name("_additional").fields(new Field[]{
+ Field.builder().name("score").build()
+ }).build()
+ })
+ .build();
+
+ WhereFilter whereFilter = WhereFilter.builder()
+ .path(new String[]{ "round" }) // Path to filter by
+ .operator(Operator.Equal)
+ .valueText("Double Jeopardy!")
+ .build();
+
+ WhereArgument whereArgument = WhereArgument.builder()
+ .filter(whereFilter)
+ .build();
+
+ String query = GetBuilder.builder()
+ .className("JeopardyQuestion")
+ .fields(fields)
+ .withBm25Filter(keyword)
+ .withWhereFilter(whereArgument)
+ .limit(3)
+ .build()
+ .buildQuery();
+
+ Result result = client.graphQL().raw().withQuery(query).run();
+ // END Filter
+ if (result.getError() != null) {
+ System.out.printf("Error inserting object: %s\n", result.getError().getMessages());
+ } else {
+ String json = new GsonBuilder().setPrettyPrinting().create().toJson(result.getResult());
+ System.out.println("The result is: " + json);
+ }
+
+ }
+}
diff --git a/_includes/code/howto/manage-data.collections-v2.ts b/_includes/code/howto/manage-data.collections-v2.ts
index 5fb3ef5aa7..9e7156f916 100644
--- a/_includes/code/howto/manage-data.collections-v2.ts
+++ b/_includes/code/howto/manage-data.collections-v2.ts
@@ -15,10 +15,10 @@ const client = weaviate.client({
},
});
-// START BasicCreateCollection // START ReadOneCollection // START UpdateCollection
+// START BasicCreateCollection // START ReadOneCollection
const className = 'Article';
-// END BasicCreateCollection // END ReadOneCollection // END UpdateCollection
+// END BasicCreateCollection // END ReadOneCollection
// ================================
// ===== CREATE A CLASS =====
@@ -482,6 +482,35 @@ Object.keys(result['moduleConfig']).includes('reranker-cohere');
await client.schema.classDeleter().withClassName(className).do();
+// ===============================================
+// ===== UPDATE THE COLLECTION WITH THE RERANKER MODULE =====
+// ===============================================
+
+const initCollectionWithReranker = {
+ class: 'Article',
+ properties: [
+ {
+ name: 'title',
+ dataType: ['text'],
+ },
+ ],
+ vectorizer: 'text2vec-openai', // this could be any vectorizer
+ // highlight-start
+ moduleConfig: {
+ 'reranker-cohere': {}, // set your reranker module
+ },
+ // highlight-end
+};
+
+// Add the class to the schema
+result = await client.schema.classCreator().withClass(initCollectionWithReranker).do();
+
+// START UpdateReranker
+// Collection definition updates are not available in the v2 API.
+// Consider upgrading to the v3 API, or deleting and recreating the collection.
+// END UpdateReranker
+
+
// ===============================================
// ===== CREATE A COLLECTION WITH A GENERATIVE MODULE =====
// ===============================================
@@ -546,6 +575,34 @@ Object.keys(result['moduleConfig']).includes('generative-openai');
// Delete the class to recreate it
await client.schema.classDeleter().withClassName(className).do();
+// ===============================================
+// ===== UPDATE THE COLLECTION WITH THE GENERATIVE MODULE =====
+// ===============================================
+
+const initCollectionWithGenerative = {
+ class: 'Article',
+ properties: [
+ {
+ name: 'title',
+ dataType: ['text'],
+ },
+ ],
+ vectorizer: 'text2vec-openai', // this could be any vectorizer
+ // highlight-start
+ moduleConfig: {
+ 'generative-openai': {}, // set your generative module
+ },
+ // highlight-end
+};
+
+// Add the class to the schema
+result = await client.schema.classCreator().withClass(initCollectionWithGenerative).do();
+
+// START UpdateGenerative
+// Collection definition updates are not available in the v2 API.
+// Consider upgrading to the v3 API, or deleting and recreating the collection.
+// END UpdateGenerative
+
// =======================
// ===== REPLICATION =====
// =======================
@@ -694,26 +751,6 @@ try {
}
// START UpdateCollection
-// Define and create a class
-const originalClassObj = {
- class: className,
- vectorIndexConfig: {
- distance: 'cosine', // Note the distance metric
- },
-};
-
-// Add the class to the schema
-let originalClassResponse = await client.schema
- .classCreator()
- .withClass(originalClassObj)
- .do();
-
-const UpdateCollectionObj = {
- class: className,
- vectorIndexConfig: {
- distance: 'dot', // Note the distance metric
- },
-};
-
-// Update the class definition
-// Not yet available in TS
+// Collection definition updates are not available in the v2 API.
+// Consider upgrading to the v3 API, or deleting and recreating the collection.
+// END UpdateCollection
diff --git a/_includes/code/howto/manage-data.collections-v3.py b/_includes/code/howto/manage-data.collections-v3.py
index 663da7c633..6169eb19ea 100644
--- a/_includes/code/howto/manage-data.collections-v3.py
+++ b/_includes/code/howto/manage-data.collections-v3.py
@@ -357,6 +357,44 @@
client.schema.delete_class(class_name)
+# ===============================================
+# ===== UPDATE THE COLLECTION WITH THE RERANKER MODULE =====
+# ===============================================
+
+# Clean slate
+if client.schema.exists(class_name):
+ client.schema.delete_class(class_name)
+
+class_obj = {
+ "class": "Article",
+ "vectorizer": "text2vec-openai", # set your vectorizer module
+ # highlight-start
+ "moduleConfig": {
+ "reranker-voyageai": {} # set your reranker module
+ }
+ # highlight-end
+}
+
+# START UpdateReranker
+class_obj = {
+ # highlight-start
+ "moduleConfig": {
+ "reranker-cohere": {} # Update your reranker module
+ }
+ # highlight-end
+}
+
+client.schema.update_config("Article", class_obj)
+# END UpdateReranker
+
+# Test
+result = client.schema.get(class_name)
+assert "reranker-cohere" in result["moduleConfig"].keys()
+
+# Delete the class to recreate it
+client.schema.delete_class(class_name)
+
+
# ===============================================
# ===== CREATE A COLLECTION WITH A GENERATIVE MODULE =====
# ===============================================
@@ -418,6 +456,46 @@
client.schema.delete_class(class_name)
+# ===============================================
+# ===== UPDATE THE COLLECTION WITH THE GENERATIVE MODULE =====
+# ===============================================
+
+# Clean slate
+if client.schema.exists(class_name):
+ client.schema.delete_class(class_name)
+
+class_obj = {
+ "class": "Article",
+ "vectorizer": "text2vec-openai", # set your vectorizer module
+ # highlight-start
+ "moduleConfig": {
+ "generative-openai": {} # set your generative module
+ }
+ # highlight-end
+}
+
+client.schema.create_class(class_obj)
+
+# START UpdateGenerative
+class_obj = {
+ "class": "Article",
+ # highlight-start
+ "moduleConfig": {
+ "generative-cohere": {} # Update your generative module
+ }
+ # highlight-end
+}
+
+client.schema.create_class(class_obj)
+# END UpdateGenerative
+
+# Test
+result = client.schema.get(class_name)
+assert "generative-cohere" in result["moduleConfig"].keys()
+
+# Delete the class to recreate it
+client.schema.delete_class(class_name)
+
# =======================
# ===== REPLICATION =====
# =======================
diff --git a/_includes/code/howto/manage-data.collections.py b/_includes/code/howto/manage-data.collections.py
index d7e3b8ae6b..9330a6b76c 100644
--- a/_includes/code/howto/manage-data.collections.py
+++ b/_includes/code/howto/manage-data.collections.py
@@ -272,6 +272,43 @@
client.collections.delete("Article")
+# ===============================================
+# ===== UPDATE A COLLECTION'S RERANKER MODULE =====
+# ===============================================
+
+client.collections.delete("Article")
+
+from weaviate.classes.config import Configure, Property, DataType
+
+client.collections.create(
+ "Article",
+ vectorizer_config=Configure.Vectorizer.text2vec_openai(),
+ # highlight-start
+ reranker_config=Configure.Reranker.voyageai()
+ # highlight-end
+)
+
+# START UpdateReranker
+from weaviate.classes.config import Reconfigure
+
+collection = client.collections.get("Article")
+
+collection.config.update(
+ # highlight-start
+ reranker_config=Reconfigure.Reranker.cohere() # Update the reranker module
+ # highlight-end
+)
+# END UpdateReranker
+
+# Test
+collection = client.collections.get("Article")
+config = collection.config.get()
+assert config.reranker_config.reranker == "reranker-cohere"
+
+# Delete the collection to recreate it
+client.collections.delete("Article")
+
+
# ===============================================
# ===== CREATE A COLLECTION WITH A GENERATIVE MODULE =====
# ===============================================
@@ -327,6 +364,43 @@
client.collections.delete("Article")
+# ===============================================
+# ===== UPDATE A COLLECTION'S GENERATIVE MODULE =====
+# ===============================================
+
+client.collections.delete("Article")
+
+from weaviate.classes.config import Configure, Property, DataType
+
+client.collections.create(
+ "Article",
+ vectorizer_config=Configure.Vectorizer.text2vec_openai(),
+ # highlight-start
+ generative_config=Configure.Generative.openai()
+ # highlight-end
+)
+
+# START UpdateGenerative
+from weaviate.classes.config import Reconfigure
+
+collection = client.collections.get("Article")
+
+collection.config.update(
+ # highlight-start
+ generative_config=Reconfigure.Generative.cohere() # Update the generative module
+ # highlight-end
+)
+# END UpdateGenerative
+
+# Test
+collection = client.collections.get("Article")
+config = collection.config.get()
+assert config.generative_config.generative == "generative-cohere"
+
+# Delete the collection to recreate it
+client.collections.delete("Article")
+
+
# ===========================
# ===== MODULE SETTINGS =====
# ===========================
@@ -491,6 +565,41 @@
client.close()
+# ==============================================
+# ===== ALL REPLICATION SETTINGS
+# ==============================================
+
+# Connect to a setting with 3 replicas
+client = weaviate.connect_to_local(
+ port=8180 # Port for demo setup with 3 replicas
+)
+
+# Clean slate
+client.collections.delete("Article")
+
+# START AllReplicationSettings
+from weaviate.classes.config import Configure, ReplicationDeletionStrategy
+
+client.collections.create(
+ "Article",
+ # highlight-start
+ replication_config=Configure.replication(
+ factor=3,
+ async_enabled=True, # Enable asynchronous repair
+ deletion_strategy=ReplicationDeletionStrategy.TIME_BASED_RESOLUTION, # Added in v1.28; Set the deletion conflict resolution strategy
+ )
+ # highlight-end
+)
+# END AllReplicationSettings
+
+# Test
+collection = client.collections.get("Article")
+config = collection.config.get()
+assert config.replication_config.async_enabled == True
+assert config.replication_config.deletion_strategy == ReplicationDeletionStrategy.TIME_BASED_RESOLUTION
+
+client.close()
+
# ====================
# ===== SHARDING =====
# ====================
diff --git a/_includes/code/howto/manage-data.collections.ts b/_includes/code/howto/manage-data.collections.ts
index a6be39cf70..25662788af 100644
--- a/_includes/code/howto/manage-data.collections.ts
+++ b/_includes/code/howto/manage-data.collections.ts
@@ -8,12 +8,14 @@ import assert from 'assert';
import weaviate, { WeaviateClient, vectorIndex } from 'weaviate-client';
import { vectorizer, reranker, generative, dataType, tokenization, configure, reconfigure, vectorDistances } from 'weaviate-client';
-const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- process.env.WCD_URL,
- {
- authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY),
- headers: {
- 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY, // Replace with your inference API key
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+const openaiKey = process.env.OPENAI_API_KEY as string
+
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
+ headers: {
+ 'X-OpenAI-Api-Key': openaiKey, // Replace with your inference API key
}
}
)
@@ -21,6 +23,14 @@ const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
const collectionName = 'Article'
let result
+
+/*
+// START UpdateCollection
+import { reconfigure } from 'weaviate-client';
+
+// END UpdateCollection
+*/
+
// START UpdateCollection // START ReadOneCollection // START ModifyParam
let articles = client.collections.get('Article')
// END UpdateCollection // END ReadOneCollection // END ModifyParam
@@ -679,18 +689,20 @@ console.log(JSON.stringify(allCollections, null, 2));
articles = client.collections.get('Article')
-/*
// START UpdateCollection
-import { reconfigure } from 'weaviate-client';
-
-// END UpdateCollection
-*/
-// START UpdateCollection
// highlight-start
-articles.config.update({
+await articles.config.update({
invertedIndex: reconfigure.invertedIndex({
- bm25k1: 1.5
+ bm25k1: 1.5 // Change the k1 parameter from 1.2
+ }),
+ vectorizers: reconfigure.vectorizer.update({
+ vectorIndexConfig: reconfigure.vectorIndex.hnsw({
+ quantizer: reconfigure.vectorIndex.quantizer.pq(),
+ ef: 4,
+ filterStrategy: 'acorn', // Available from Weaviate v1.27.0
+ }),
+
})
})
// highlight-end
diff --git a/_includes/code/howto/manage-data.import-v3.py b/_includes/code/howto/manage-data.import-v3.py
index de01dc686b..d13f68d8fb 100644
--- a/_includes/code/howto/manage-data.import-v3.py
+++ b/_includes/code/howto/manage-data.import-v3.py
@@ -172,7 +172,7 @@
# Settings for displaying the import progress
counter = 0
-interval = 20 # print progress every this many records; should be bigger than the batch_size
+interval = 200 # print progress every this many records; should be bigger than the batch_size
def add_object(obj) -> None:
global counter
diff --git a/_includes/code/howto/manage-data.import.py b/_includes/code/howto/manage-data.import.py
index 7b3517cc5e..3398c71719 100644
--- a/_includes/code/howto/manage-data.import.py
+++ b/_includes/code/howto/manage-data.import.py
@@ -344,10 +344,10 @@
# Settings for displaying the import progress
counter = 0
-interval = 100 # print progress every this many records; should be bigger than the batch_size
+interval = 200 # print progress every this many records; should be bigger than the batch_size
print("JSON streaming, to avoid running out of memory on large files...")
-with client.batch.fixed_size(batch_size=200) as batch:
+with client.batch.fixed_size(batch_size=100) as batch:
with open("jeopardy_1k.json", "rb") as f:
objects = ijson.items(f, "item")
for obj in objects:
@@ -393,7 +393,7 @@
# Settings for displaying the import progress
counter = 0
-interval = 100 # print progress every this many records; should be bigger than the batch_size
+interval = 200 # print progress every this many records; should be bigger than the batch_size
def add_object(obj) -> None:
global counter
@@ -402,7 +402,7 @@ def add_object(obj) -> None:
"answer": obj["Answer"],
}
- with client.batch.fixed_size(batch_size=200) as batch:
+ with client.batch.fixed_size(batch_size=100) as batch:
batch.add_object(
collection="JeopardyQuestion",
properties=properties,
diff --git a/_includes/code/howto/manage-data.multi-tenancy.ts b/_includes/code/howto/manage-data.multi-tenancy.ts
index e7a874a945..fa428c1180 100644
--- a/_includes/code/howto/manage-data.multi-tenancy.ts
+++ b/_includes/code/howto/manage-data.multi-tenancy.ts
@@ -3,12 +3,20 @@
import assert from 'assert';
import weaviate, { WeaviateClient } from 'weaviate-client';
+/*
+// START UpdateAutoMT
+import { reconfigure } from 'weaviate-client';
+
+// END UpdateAutoMT
+*/
+
+
const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- process.env.WCD_URL,
+ process.env.WCD_URL as string,
{
- authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY),
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
headers: {
- 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY, // Replace with your inference API key
+ 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, // Replace with your inference API key
}
}
)
@@ -66,9 +74,16 @@ const result = await client.collections.create({
// ==========================
{
-// START enable autoMT
-// Coming soon
-// END enable autoMT
+// START UpdateAutoMT
+const multiCollection = client.collections.get('MultiTenancyCollection');
+
+await multiCollection.config.update({
+ multiTenancy: weaviate.reconfigure.multiTenancy({
+ autoTenantActivation: true,
+ autoTenantCreation: true,
+ }),
+});
+// END UpdateAutoMT
}
// ================================
@@ -117,8 +132,8 @@ assert.ok(['tenantA', 'tenantB'].includes(tenants[1].name));
// ===== Get tenants from a collection by name =====
// =======================================
{
-// START GetTenantsByName
-const multiCollection = client.collections.get('MultiTenancyCollection');
+// START GetTenantsByName
+const multiCollection = client.collections.get('MultiTenancyCollection');
// highlight-start
const tenants = await multiCollection.tenants.getByNames(['tenantA', 'tenantB'])
@@ -133,7 +148,7 @@ console.log(tenants)
{
// START GetOneTenant
-const multiCollection = client.collections.get('MultiTenancyCollection');
+const multiCollection = client.collections.get('MultiTenancyCollection');
// highlight-start
const tenant = await multiCollection.tenants.getByName('tenantA')
@@ -268,7 +283,7 @@ await multiTenantA.data.insert({
// =====================
{
// START Search
-const multiCollection = client.collections.get('MultiTenancyCollection');
+const multiCollection = client.collections.get('MultiTenancyCollection');
// highlight-start
const multiTenantA = multiCollection.withTenant('tenantA')
diff --git a/_includes/code/howto/search.aggregate-v2.ts b/_includes/code/howto/search.aggregate-v2.ts
index 01226d0e9c..cc63517418 100644
--- a/_includes/code/howto/search.aggregate-v2.ts
+++ b/_includes/code/howto/search.aggregate-v2.ts
@@ -169,6 +169,21 @@ assert.deepEqual(result.data.Aggregate.JeopardyQuestion[0], { points: { sum: 300
// End test
+// =========================================
+// ===== Hybrid EXAMPLES =====
+// =========================================
+
+// HybridExample
+// TS client support coming soon
+// END HybridExample
+
+// Test
+assert('JeopardyQuestion' in result.data.Aggregate);
+assert.equal(result.data.Aggregate.JeopardyQuestion.length, 1);
+assert.deepEqual(result.data.Aggregate.JeopardyQuestion[0], { points: { sum: 4600 } });
+// End test
+
+
// =================================
// ===== where filter EXAMPLES =====
// =================================
@@ -195,4 +210,4 @@ console.log(JSON.stringify(result, null, 2));
assert('JeopardyQuestion' in result.data.Aggregate);
assert.equal(result.data.Aggregate.JeopardyQuestion.length, 1);
assert.deepEqual(result.data.Aggregate.JeopardyQuestion[0], { meta: { count: 285 } });
-// End test
\ No newline at end of file
+// End test
diff --git a/_includes/code/howto/search.aggregate-v3.py b/_includes/code/howto/search.aggregate-v3.py
index 5b514261bb..40da416bb8 100644
--- a/_includes/code/howto/search.aggregate-v3.py
+++ b/_includes/code/howto/search.aggregate-v3.py
@@ -439,6 +439,72 @@
)
+# =========================================
+# ===== nearTextWithLimit EXAMPLES =====
+# =========================================
+
+# HybridExample
+response = (
+ client.query
+ .aggregate("JeopardyQuestion")
+ .with_hybrid({
+ "query": "animals in space"
+ })
+ # highlight-start
+ .with_object_limit(10)
+ # highlight-end
+ .with_fields("points { sum }")
+ .do()
+)
+print(json.dumps(response, indent=2))
+# END HybridExample
+
+
+gql_query = """
+# GraphQLHybridExample
+{
+ Aggregate {
+ JeopardyQuestion(
+ hybrid: {
+ query: "animals in space"
+ }
+ # highlight-start
+ objectLimit: 10
+ # highlight-end
+ ) {
+ points {
+ sum
+ }
+ }
+ }
+}
+# END GraphQLHybridExample
+"""
+gqlresponse = client.query.raw(gql_query)
+
+# Test results
+assert gqlresponse == response
+# END Test results
+
+expected_response = (
+# ResultsHybridExample
+{
+ "data": {
+ "Aggregate": {
+ "JeopardyQuestion": [
+ {
+ "points": {
+ "sum": 6700
+ }
+ }
+ ]
+ }
+ }
+}
+# END ResultsHybridExample
+)
+
+
# =================================
# ===== where filter EXAMPLES =====
# =================================
diff --git a/_includes/code/howto/search.aggregate.py b/_includes/code/howto/search.aggregate.py
index b685205d79..cdce8b45f4 100644
--- a/_includes/code/howto/search.aggregate.py
+++ b/_includes/code/howto/search.aggregate.py
@@ -142,6 +142,32 @@
# End test
+
+# =========================================
+# ===== hybrid EXAMPLES =====
+# =========================================
+
+# HybridExample
+from weaviate.classes.query import Metrics
+
+jeopardy = client.collections.get("JeopardyQuestion")
+response = jeopardy.aggregate.hybrid(
+ query="animals in space",
+ # highlight-start
+ object_limit=10,
+ # highlight-end
+ return_metrics=Metrics("points").number(sum_=True),
+)
+
+print(response.properties["points"].sum_)
+# END HybridExample
+
+
+# Tests
+assert response.properties["points"].sum_ > 0
+# End test
+
+
# ============================
# ===== nearTextWithDistance EXAMPLES =====
# ============================
diff --git a/_includes/code/howto/search.aggregate.ts b/_includes/code/howto/search.aggregate.ts
index 1028c4e360..b3924ae3b6 100644
--- a/_includes/code/howto/search.aggregate.ts
+++ b/_includes/code/howto/search.aggregate.ts
@@ -4,13 +4,13 @@ import assert from 'assert';
import weaviate, { WeaviateClient } from 'weaviate-client';
const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- process.env.WCD_URL,
+ process.env.WCD_URL as string,
{
- authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY),
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
headers: {
- 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY, // Replace with your inference API key
+ 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, // Replace with your inference API key
}
- }
+ }
)
// MetaCount TS // TextProp TS // IntProp TS // groupBy TS // nearTextWithLimit TS // nearTextWithDistance TS // whereFilter TS
@@ -94,7 +94,7 @@ console.log(JSON.stringify(result.properties, null, 2));
// ============================
{
// groupBy TS
-
+
const result = await jeopardy.aggregate.groupBy.overAll({
// highlight-start
groupBy: { property: 'round' }
@@ -120,7 +120,7 @@ console.log(JSON.stringify(result, null, 2));
// =========================================
{
// nearTextWithLimit TS
-
+
const result = await jeopardy.aggregate.nearText('animals in space', {
// highlight-start
objectLimit: 10,
@@ -143,15 +143,15 @@ console.log(result.properties['points'].sum);
// ============================
{
// nearTextWithDistance TS
-
+
const result = await jeopardy.aggregate.nearText(['animals in space'],{
// highlight-start
distance: 0.19,
// highlight-end
- returnMetrics: jeopardy.metrics.aggregate('points').number(['sum'])
+ returnMetrics: jeopardy.metrics.aggregate('points').number(["sum"])
})
-console.log(result.properties['points'].sum);
+console.log(result.properties['points']);
// END nearTextWithDistance TS
// Test
@@ -161,12 +161,27 @@ console.log(result.properties['points'].sum);
// End test
}
+// =========================================
+// ===== Hybrid EXAMPLES =====
+// =========================================
+{
+// HybridExample
+// TS client support coming soon
+// END HybridExample
+
+ // Test
+ // assert('JeopardyQuestion' in result.data.Aggregate);
+ // assert.equal(result.data.Aggregate.JeopardyQuestion.length, 1);
+ // assert.deepEqual(result.data.Aggregate.JeopardyQuestion[0], { points: { sum: 4600 } });
+ // End test
+ }
+
// =================================
// ===== where filter EXAMPLES =====
// =================================
{
// whereFilter TS
-
+
const result = await jeopardy.aggregate.overAll({
// highlight-start
filters: jeopardy.filter.byProperty('round').equal('Final Jeopardy!')
diff --git a/_includes/code/howto/search.hybrid-v3.py b/_includes/code/howto/search.hybrid-v3.py
index 444fd2c28e..98af3218f0 100644
--- a/_includes/code/howto/search.hybrid-v3.py
+++ b/_includes/code/howto/search.hybrid-v3.py
@@ -10,10 +10,10 @@
# Instantiate the client with the user/password and OpenAI api key
client = weaviate.Client(
- "https://edu-demo.weaviate.network",
- auth_client_secret=weaviate.auth.AuthApiKey("learn-weaviate"),
+ os.getenv("WCD_DEMO_URL"), # Replace with your Weaviate URL
+ auth_client_secret=weaviate.auth.AuthApiKey(os.getenv("WCD_DEMO_RO_KEY")), # If authentication is on. Replace with your Weaviate instance API key
additional_headers={
- "X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]
+ "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY") # Replace with your OPENAI API key
}
)
diff --git a/_includes/code/howto/search.hybrid.py b/_includes/code/howto/search.hybrid.py
index 85da3a0d0e..cb596e9ee6 100644
--- a/_includes/code/howto/search.hybrid.py
+++ b/_includes/code/howto/search.hybrid.py
@@ -302,10 +302,10 @@
# End test
# =========================================
-# ===== Hybrid with vector similarity =====
+# ===== Hybrid with vector parameters =====
# =========================================
-# START VectorSimilarityPython
+# START VectorParametersPython
from weaviate.classes.query import HybridVector, Move, HybridFusion
jeopardy = client.collections.get("JeopardyQuestion")
@@ -321,6 +321,27 @@
alpha=0.75,
limit=5,
)
+# END VectorParametersPython
+
+assert len(response.objects) <= 5
+assert len(response.objects) > 0
+
+# =========================================
+# ===== Hybrid with vector similarity threshold =====
+# =========================================
+
+# START VectorSimilarityPython
+from weaviate.classes.query import HybridVector, Move, HybridFusion
+
+jeopardy = client.collections.get("JeopardyQuestion")
+response = jeopardy.query.hybrid(
+ query="California",
+ # highlight-start
+ max_vector_distance=0.4, # Maximum threshold for the vector search component
+ # highlight-end
+ alpha=0.75,
+ limit=5,
+)
# END VectorSimilarityPython
assert len(response.objects) <= 5
diff --git a/_includes/code/howto/search.hybrid.ts b/_includes/code/howto/search.hybrid.ts
index 79f1c429aa..ad72d3394a 100644
--- a/_includes/code/howto/search.hybrid.ts
+++ b/_includes/code/howto/search.hybrid.ts
@@ -9,11 +9,11 @@ import assert from 'assert';
import weaviate from 'weaviate-client';
const client = await weaviate.connectToWeaviateCloud(
- process.env.WCD_URL,
+ process.env.WCD_URL as string,
{
- authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY),
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
headers: {
- 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY, // Replace with your inference API key
+ 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, // Replace with your inference API key
}
}
)
@@ -235,7 +235,8 @@ const result = await jeopardy.query.hybrid('California', {
},
// maxVectorDistance support coming soon
alpha: 0.75,
- limit: 5
+ limit: 5,
+
})
for (let object of result.objects) {
diff --git a/_includes/code/howto/search.image.ts b/_includes/code/howto/search.image.ts
index 60c7168073..e5962f5f85 100644
--- a/_includes/code/howto/search.image.ts
+++ b/_includes/code/howto/search.image.ts
@@ -7,27 +7,27 @@
import assert from 'assert';
+let result
// =================================================
// ===== Helper functions to convert to base64 =====
// =================================================
// START helper base64 functions
-import { readFileSync } from 'fs'
+const imageURL = 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/14/Deutsches_Museum_Portrait_4.jpg/500px-Deutsches_Museum_Portrait_4.jpg'
-const urlToBase64 = async (imageUrl) => {
+async function urlToBase64(imageUrl: string) {
const response = await fetch(imageUrl);
const content = await response.buffer();
return content.toString('base64');
}
-urlToBase64('https://upload.wikimedia.org/wikipedia/commons/thumb/1/14/Deutsches_Museum_Portrait_4.jpg/500px-Deutsches_Museum_Portrait_4.jpg')
- .then(base64 => { console.log(base64) });
-
+const base64 = await urlToBase64(imageURL)
+console.log(base64)
// END helper base64 functions
- // Helper function – get base64 representation from a local file
+// Helper function – get base64 representation from a local file
// This example is for NodeJS
const fileToBase64 = (file: string) => {
return readFileSync(file, { encoding: 'base64' });
@@ -44,12 +44,12 @@ import fs from 'fs';
const client = await weaviate.connectToWeaviateCloud(
'WCD-URL',
- {
- authCredentials: new weaviate.ApiKey('WEAVIATE-API-KEY'),
- headers: {
- 'X-OpenAI-Api-Key': 'OPENAI-API-KEY', // Replace with your inference API key
- }
- }
+ {
+ authCredentials: new weaviate.ApiKey('WEAVIATE-API-KEY'),
+ headers: {
+ 'X-OpenAI-Api-Key': 'OPENAI-API-KEY', // Replace with your inference API key
+ }
+ }
)
const imageUrl = 'https://upload.wikimedia.org/wikipedia/commons/thumb/f/fb/Welchcorgipembroke.JPG/640px-Welchcorgipembroke.JPG'
@@ -59,27 +59,55 @@ const response = await fetch(imageUrl);
const content = await response.buffer();
{
-// START search with base64
-// Perform query
-const myCollection = client.collections.get('Dog');
+ // START search with base64
+ import { toBase64FromMedia } from 'weaviate-client';
-// highlight-start
-const base64String = 'SOME_BASE_64_REPRESENTATION';
-// highlight-end
+ const myCollection = client.collections.get('Dog');
+ const filePath = './images/search-image.jpg'
+ // highlight-start
+ const base64String = await toBase64FromMedia(file.path)
+ // highlight-end
-// highlight-start
-const result = await myCollection.query.nearImage(base64String, {
-// highlight-end
- returnProperties: ['breed'],
- limit: 1,
- // targetVector: 'vector_name' // required when using multiple named vectors
-})
+ // Perform query
+ // highlight-start
+ const result = await myCollection.query.nearImage(base64String, {
+ returnProperties: ['breed'],
+ limit: 1,
+ // targetVector: 'vector_name' // required when using multiple named vectors
+ })
+ // highlight-end
-console.log(JSON.stringify(result.objects, null, 2));
-// END search with base64
+ console.log(JSON.stringify(result.objects, null, 2));
+ // END search with base64
-// Tests
-// assert.deepEqual(result.data['Get']['Dog'], [{ 'breed': 'Corgi' }]);
+ // Tests
+ // assert.deepEqual(result.data['Get']['Dog'], [{ 'breed': 'Corgi' }]);
+}
+
+{
+ // START search with Blob
+ // Perform query
+ const myCollection = client.collections.get('Dog');
+
+ // highlight-start
+ const url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/f/fb/Welchcorgipembroke.JPG/640px-Welchcorgipembroke.JPG'
+ const response = await fetch(url);
+ const blob = await response.buffer();
+ // highlight-end
+
+ // highlight-start
+ const result = await myCollection.query.nearImage(blob, {
+ // highlight-end
+ returnProperties: ['breed'],
+ limit: 1,
+ // targetVector: 'vector_name' // required when using multiple named vectors
+ })
+
+ console.log(JSON.stringify(result.objects, null, 2));
+ // END search with Blob
+
+ // Tests
+ // assert.deepEqual(result.data['Get']['Dog'], [{ 'breed': 'Corgi' }]);
}
// ====================================
@@ -87,23 +115,18 @@ console.log(JSON.stringify(result.objects, null, 2));
// ====================================
fs.writeFileSync('image.jpg', content);
{
-// START ImageFileSearch
-const myCollection = client.collections.get('Dog');
-
-// highlight-start
-// Read the file
-const image = await fs.promises.readFile('image.jpg');
-// highlight-end
-
-// Query based on the image content
-const result = await myCollection.query.nearImage(image, {
- returnProperties: ['breed'],
- limit: 1,
- // targetVector: 'vector_name' // required when using multiple named vectors
-})
-
-console.log(JSON.stringify(result.objects, null, 2));
-// END ImageFileSearch
+ // START ImageFileSearch
+ const myCollection = client.collections.get('Dog');
+
+ // Query based on the image content
+ const result = await myCollection.query.nearImage('./images/search-image.jpg', {
+ returnProperties: ['breed'],
+ limit: 1,
+ // targetVector: 'vector_name' // required when using multiple named vectors
+ })
+
+ console.log(JSON.stringify(result.objects, null, 2));
+ // END ImageFileSearch
}
// Tests
@@ -134,3 +157,15 @@ console.log(JSON.stringify(result, null, 2));
// Tests
assert.equal(result.data['Get']['Dog'][0]['breed'], 'Corgi');
+
+const displayRes = `
+// START Expected base64 results
+[{
+ metadata: {},
+ properties: { name: '000ada55d36b4bcb.jpg' },
+ references: undefined,
+ uuid: 'd2528234-1576-4033-b17f-081fe051bee0',
+ vectors: {}
+}]
+// END Expected base64 results
+`
diff --git a/_includes/code/howto/search.multi-target-v3.ts b/_includes/code/howto/search.multi-target-v3.ts
index 54f78e0d06..798035acd5 100644
--- a/_includes/code/howto/search.multi-target-v3.ts
+++ b/_includes/code/howto/search.multi-target-v3.ts
@@ -34,7 +34,7 @@ result = await jeopardy.query.nearText('a wild animal', {
result.objects.forEach((item) => {
console.log(JSON.stringify(item.properties, null, 2));
- console.log(JSON.stringify(item.metadata.distance, null, 2));
+ console.log(JSON.stringify(item.metadata?.distance, null, 2));
});
// END MultiBasic
@@ -49,6 +49,8 @@ var resp = await jeopardy.query.fetchObjects({
var v1 = resp.objects[0].vectors.jeopardy_questions_vector;
var v2 = resp.objects[0].vectors.jeopardy_answers_vector;
+var v3 = resp.objects[1].vectors.jeopardy_answers_vector;
+
// START MultiTargetNearVector
jeopardy = client.collections.get('JeopardyTiny');
@@ -68,7 +70,7 @@ result = await jeopardy.query.nearVector({
result.objects.forEach((item) => {
console.log(JSON.stringify(item.properties, null, 2));
- console.log(JSON.stringify(item.metadata.distance, null, 2));
+ console.log(JSON.stringify(item.metadata?.distance, null, 2));
});
// END MultiTargetNearVector
@@ -89,7 +91,7 @@ result = await jeopardy.query.nearText('a wild animal', {
result.objects.forEach((item) => {
console.log(JSON.stringify(item.properties, null, 2));
- console.log(JSON.stringify(item.metadata.distance, null, 2));
+ console.log(JSON.stringify(item.metadata?.distance, null, 2));
});
// END MultiTargetWithSimpleJoin
@@ -113,7 +115,7 @@ result = await jeopardy.query.nearText('a wild animal', {
result.objects.forEach((item) => {
console.log(JSON.stringify(item.properties, null, 2));
- console.log(JSON.stringify(item.metadata.distance, null, 2));
+ console.log(JSON.stringify(item.metadata?.distance, null, 2));
});
// END MultiTargetManualWeights
@@ -137,8 +139,56 @@ result = await jeopardy.query.nearText('a wild animal', {
result.objects.forEach((item) => {
console.log(JSON.stringify(item.properties, null, 2));
- console.log(JSON.stringify(item.metadata.distance, null, 2));
+ console.log(JSON.stringify(item.metadata?.distance, null, 2));
});
// END MultiTargetRelativeScore
+
+// START MultiTargetMultipleNearVectorsV1
+jeopardy = client.collections.get('JeopardyTiny');
+
+result = await jeopardy.query.nearVector({
+ // highlight-start
+ // Specify the query vectors for each target vector. where v1, v2.. are vectors
+ 'jeopardy_questions_vector': v1,
+ 'jeopardy_answers_vector': [v2, v3]
+ // highlight-end
+}, {
+ limit: 2,
+ // Specify the target vectors as a list
+ targetVector: ['jeopardy_questions_vector', 'jeopardy_answers_vector'],
+ returnMetadata: ['distance'],
+});
+
+result.objects.forEach((item) => {
+ console.log(JSON.stringify(item.properties, null, 2));
+ console.log(JSON.stringify(item.metadata?.distance, null, 2));
+});
+// END MultiTargetMultipleNearVectorsV1
+
+// START MultiTargetMultipleNearVectorsV2
+jeopardy = client.collections.get('JeopardyTiny');
+
+result = await jeopardy.query.nearVector({
+ // highlight-start
+ 'jeopardy_questions_vector': v1,
+ 'jeopardy_answers_vector': [v2, v3]
+}, {
+ limit: 2,
+ // Specify the target vectors as a list
+ targetVector: jeopardy.multiTargetVector.manualWeights({
+ "jeopardy_questions_vector": 10,
+ "jeopardy_answers_vector": [30, 30], // Matches the order of the vectors above
+ }),
+ // highlight-end
+ returnMetadata: ['distance'],
+});
+
+result.objects.forEach((item) => {
+ console.log(JSON.stringify(item.properties, null, 2));
+ console.log(JSON.stringify(item.metadata?.distance, null, 2));
+});
+// END MultiTargetMultipleNearVectorsV2
+
+
client.close();
diff --git a/_includes/code/howto/search.multi-target-v4.py b/_includes/code/howto/search.multi-target-v4.py
index 1d24a21bc6..af8671ffa4 100644
--- a/_includes/code/howto/search.multi-target-v4.py
+++ b/_includes/code/howto/search.multi-target-v4.py
@@ -167,7 +167,7 @@
# END MultiTargetMultipleNearVectorsV1
# START MultiTargetMultipleNearVectorsV2
-from weaviate.classes.query import MetadataQuery
+from weaviate.classes.query import TargetVectors, MetadataQuery
collection = client.collections.get("JeopardyTiny")
diff --git a/_includes/code/python/howto.configure.rbac.permissions.py b/_includes/code/python/howto.configure.rbac.permissions.py
new file mode 100644
index 0000000000..baeab65d3b
--- /dev/null
+++ b/_includes/code/python/howto.configure.rbac.permissions.py
@@ -0,0 +1,95 @@
+import weaviate
+from weaviate import WeaviateClient
+
+# START-ANY
+from weaviate.classes.rbac import Permissions
+
+# END-ANY
+from weaviate.classes.init import Auth
+
+admin_client = weaviate.connect_to_local(
+ # Use custom port defined in tests/docker-compose-rbac.yml (without showing the user)
+ port=8580,
+ grpc_port=50551,
+ auth_credentials=Auth.api_key("admin-key")
+)
+
+def reset_user(user: str, client: WeaviateClient):
+ # Clean slate
+ current_roles = client.roles.by_user(user) # check if user exists
+ for k in current_roles.keys():
+ client.roles.revoke_from_user(role_names=k, user=user) # revoke all roles
+
+# =================================================================
+# =============== EXAMPLE: READ + WRITE PERMISSIONS
+# =================================================================
+
+# Clean slate
+reset_user("other-user", client=admin_client)
+admin_client.roles.delete("rw_role_target_collections") # delete if exists
+
+# START ReadWritePermissionDefinition
+# Define permissions (example confers read+write rights to collections starting with "TargetCollection_")
+admin_permissions = [
+ Permissions.collections(
+ collection="TargetCollection_*",
+ create_collection=True,
+ read_config=True,
+ update_config=True,
+ delete_collection=True,
+ ),
+ Permissions.data(
+ collection="TargetCollection_*",
+ create=True,
+ read=True,
+ update=True,
+ delete=True
+ ),
+ Permissions.backup(collection="TargetCollection_*", manage=True),
+ Permissions.nodes(collection="TargetCollection_*", read=True),
+ Permissions.cluster(read=True),
+]
+
+# Create a new role and assign it to a user
+admin_client.roles.create(role_name="rw_role_target_collections", permissions=admin_permissions)
+admin_client.roles.assign_to_user(role_names="rw_role_target_collections", user="other-user")
+# END ReadWritePermissionDefinition
+
+# ===== TEST ===== basic checks to see if the role was created
+user_permissions = admin_client.roles.by_user("other-user")
+
+assert "rw_role_target_collections" in user_permissions.keys()
+assert user_permissions["rw_role_target_collections"].collections_permissions[0].collection == "TargetCollection_*"
+assert user_permissions["rw_role_target_collections"].name == "rw_role_target_collections"
+
+# =================================================================
+# =============== EXAMPLE: VIEWER PERMISSIONS
+# =================================================================
+
+# Clean slate
+reset_user("other-user", client=admin_client)
+admin_client.roles.delete("viewer_role_target_collections") # delete if exists
+
+# START ViewerPermissionDefinition
+# Define permissions (example confers viewer rights to collections starting with "TargetCollection_")
+viewer_permissions = [
+ Permissions.collections(
+ collection="TargetCollection_*",
+ read_config=True,
+ ),
+ Permissions.data(collection="TargetCollection_*", read=True),
+]
+
+# Create a new role and assign it to a user
+admin_client.roles.create(role_name="viewer_role_target_collections", permissions=viewer_permissions)
+admin_client.roles.assign_to_user(role_names="viewer_role_target_collections", user="other-user")
+# END ViewerPermissionDefinition
+
+# ===== TEST ===== basic checks to see if the role was created
+user_permissions = admin_client.roles.by_user("other-user")
+
+assert "viewer_role_target_collections" in user_permissions.keys()
+assert user_permissions["viewer_role_target_collections"].collections_permissions[0].collection == "TargetCollection_*"
+assert user_permissions["viewer_role_target_collections"].name == "viewer_role_target_collections"
+
+admin_client.close()
diff --git a/_includes/code/python/howto.configure.rbac.roles.py b/_includes/code/python/howto.configure.rbac.roles.py
new file mode 100644
index 0000000000..fc2df06eb4
--- /dev/null
+++ b/_includes/code/python/howto.configure.rbac.roles.py
@@ -0,0 +1,152 @@
+from weaviate.classes.rbac import Permissions
+
+# START AdminClient
+import weaviate
+from weaviate.classes.init import Auth
+
+# Connect to Weaviate as Admin
+admin_client = weaviate.connect_to_local(
+ # END AdminClient
+ # Use custom port defined in tests/docker-compose-rbac.yml (without showing the user)
+ port=8580,
+ grpc_port=50551,
+ # START AdminClient
+ auth_credentials=Auth.api_key("admin-key")
+)
+# END AdminClient
+
+jane_client = weaviate.connect_to_local(
+ port=8580, grpc_port=50551, auth_credentials=Auth.api_key("jane-key")
+)
+
+# # START CreateRole
+# admin_client.roles.create(role_name="devrel")
+# # END CreateRole
+
+# START AddRoleAtCreate
+from weaviate.classes.rbac import Permissions
+
+permissions = [
+ Permissions.collections(
+ collection="Test_*",
+ read_config=True,
+ create_collection=False,
+ delete_collection=False,
+ ),
+ Permissions.collections(
+ collection="Test_DevRel",
+ read_config=True,
+ create_collection=True,
+ delete_collection=True,
+ ),
+]
+
+admin_client.roles.create(
+ role_name="devrel", permissions=permissions
+)
+# END AddRoleAtCreate
+assert "devrel" in admin_client.roles.list_all().keys()
+
+# START AddRoles
+from weaviate.classes.rbac import Permissions
+
+permissions = [
+ Permissions.data(collection="Test_DevRel", read=True, create=True),
+ Permissions.data(collection="Test_*", read=True, create=False)
+]
+
+admin_client.roles.add_permissions(permissions=permissions, role_name="devrel")
+# END AddRoles
+
+# START ManageRoles
+from weaviate.classes.rbac import Permissions
+
+permissions = [
+ Permissions.roles(role="devrel", read=True, manage=True),
+ Permissions.roles(role="devrel-*", read=True, manage=False)
+]
+
+admin_client.roles.create(role_name="devrel-admin", permissions=permissions)
+# END ManageRoles
+
+# START ClusterAndNodePermissions
+from weaviate.classes.rbac import Permissions
+
+permissions = [
+ Permissions.cluster(read=True),
+ Permissions.nodes(collection="Test_DevRel", verbosity="verbose", read=True)
+]
+
+admin_client.roles.add_permissions(permissions=permissions, role_name="devrel-admin")
+# END ClusterAndNodePermissions
+
+
+# START AssignRole
+admin_client.roles.assign_to_user(role_names="devrel", user="jane-doe")
+# END AssignRole
+assert "devrel" in admin_client.roles.by_user(user="jane-doe")
+
+# START ListCurrentUserRoles
+print(admin_client.roles.of_current_user())
+# END ListCurrentUserRoles
+
+# START ListUserRoles
+user_roles = admin_client.roles.by_user(user="jane-doe")
+
+for role in user_roles:
+ print(role)
+# END ListUserRoles
+assert any(permission.collection == "Test_DevRel" for permission in user_roles["devrel"].collections_permissions)
+
+# START CheckRoleExists
+print(admin_client.roles.exists(role_name="role-name")) # Returns True or False
+# END CheckRoleExists
+
+# START InspectRole
+print(admin_client.roles.by_name(role_name="devrel"))
+# END InspectRole
+
+# START AssignedUsers
+assigned_users = admin_client.roles.assigned_users(role_name="devrel")
+
+for user in assigned_users:
+ print(user)
+# END AssignedUsers
+assert "jane-doe" in assigned_users
+
+# START ListAllRoles
+all_roles = admin_client.roles.list_all()
+
+for role_name, role in all_roles.items():
+ print(role_name, role)
+# END ListAllRoles
+
+# START RemovePermissions
+from weaviate.classes.rbac import Permissions
+
+permissions = [
+ Permissions.collections(
+ collection="Test_DevRel",
+ read_config=True,
+ create_collection=True,
+ delete_collection=True,
+ ),
+ Permissions.data(collection="Test_*", read=True, create=False)
+]
+
+admin_client.roles.remove_permissions(
+ role_name="devrel", permissions=permissions
+)
+# END RemovePermissions
+
+# START RevokeRoles
+admin_client.roles.revoke_from_user(role_names=["devrel"], user="jane-doe")
+# END RevokeRoles
+assert "devrel" not in admin_client.roles.by_user(user="jane-doe")
+
+# START DeleteRole
+admin_client.roles.delete(role_name="devrel")
+# END DeleteRole
+
+admin_client.close()
+jane_client.close()
diff --git a/_includes/code/python/local.quickstart.create_collection.py b/_includes/code/python/local.quickstart.create_collection.py
new file mode 100644
index 0000000000..490360ec5a
--- /dev/null
+++ b/_includes/code/python/local.quickstart.create_collection.py
@@ -0,0 +1,31 @@
+# CreateCollection
+import weaviate
+from weaviate.classes.config import Configure
+
+client = weaviate.connect_to_local()
+
+# END CreateCollection
+
+# NOT SHOWN TO THE USER - DELETE EXISTING COLLECTION
+client.collections.delete("Question")
+
+# CreateCollection
+# highlight-start
+questions = client.collections.create(
+ name="Question",
+ vectorizer_config=Configure.Vectorizer.text2vec_ollama( # Configure the Ollama embedding integration
+ api_endpoint="http://host.docker.internal:11434", # Allow Weaviate from within a Docker container to contact your Ollama instance
+ model="nomic-embed-text", # The model to use
+ ),
+ generative_config=Configure.Generative.ollama( # Configure the Ollama generative integration
+ api_endpoint="http://host.docker.internal:11434", # Allow Weaviate from within a Docker container to contact your Ollama instance
+ model="llama3.2", # The model to use
+ )
+)
+# highlight-end
+# CreateCollection
+
+client.close() # Free up resources
+# END CreateCollection
+
+
diff --git a/_includes/code/python/local.quickstart.import_objects.py b/_includes/code/python/local.quickstart.import_objects.py
new file mode 100644
index 0000000000..f50480ecc1
--- /dev/null
+++ b/_includes/code/python/local.quickstart.import_objects.py
@@ -0,0 +1,25 @@
+# Import
+import weaviate
+import requests, json
+
+client = weaviate.connect_to_local()
+
+resp = requests.get(
+ "https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/jeopardy_tiny.json"
+)
+data = json.loads(resp.text)
+
+# highlight-start
+questions = client.collections.get("Question")
+
+with questions.batch.dynamic() as batch:
+ for d in data:
+ batch.add_object({
+ "answer": d["Answer"],
+ "question": d["Question"],
+ "category": d["Category"],
+ })
+# highlight-end
+
+client.close() # Free up resources
+# END Import
diff --git a/_includes/code/python/local.quickstart.is_ready.py b/_includes/code/python/local.quickstart.is_ready.py
new file mode 100644
index 0000000000..878f8b4bc8
--- /dev/null
+++ b/_includes/code/python/local.quickstart.is_ready.py
@@ -0,0 +1,11 @@
+# InstantiationExample
+import weaviate
+
+# highlight-start
+client = weaviate.connect_to_local()
+# highlight-end
+
+print(client.is_ready()) # Should print: `True`
+
+client.close() # Free up resources
+# END InstantiationExample
diff --git a/_includes/code/python/local.quickstart.query.neartext.py b/_includes/code/python/local.quickstart.query.neartext.py
new file mode 100644
index 0000000000..9fe80928ad
--- /dev/null
+++ b/_includes/code/python/local.quickstart.query.neartext.py
@@ -0,0 +1,20 @@
+# NearText
+import weaviate
+import json
+
+client = weaviate.connect_to_local()
+
+questions = client.collections.get("Question")
+
+# highlight-start
+response = questions.query.near_text(
+ query="biology",
+ limit=2
+)
+# highlight-end
+
+for obj in response.objects:
+ print(json.dumps(obj.properties, indent=2))
+
+client.close() # Free up resources
+# END NearText
diff --git a/_includes/code/python/local.quickstart.query.rag.py b/_includes/code/python/local.quickstart.query.rag.py
new file mode 100644
index 0000000000..504bd929f9
--- /dev/null
+++ b/_includes/code/python/local.quickstart.query.rag.py
@@ -0,0 +1,19 @@
+# RAG
+import weaviate
+
+client = weaviate.connect_to_local()
+
+questions = client.collections.get("Question")
+
+# highlight-start
+response = questions.generate.near_text(
+ query="biology",
+ limit=2,
+ grouped_task="Write a tweet with emojis about these facts."
+)
+# highlight-end
+
+print(response.generated) # Inspect the generated text
+
+client.close() # Free up resources
+# END RAG
diff --git a/_includes/code/python/quickstart.create_collection.py b/_includes/code/python/quickstart.create_collection.py
index 20632f1511..5c0f8c7dfd 100644
--- a/_includes/code/python/quickstart.create_collection.py
+++ b/_includes/code/python/quickstart.create_collection.py
@@ -14,6 +14,7 @@
)
# END CreateCollection
+
# NOT SHOWN TO THE USER - DELETE EXISTING COLLECTION
client.collections.delete("Question")
@@ -21,8 +22,8 @@
# highlight-start
questions = client.collections.create(
name="Question",
- vectorizer_config=Configure.Vectorizer.text2vec_openai(), # Configure the OpenAI embedding integration
- generative_config=Configure.Generative.openai() # Configure the OpenAI generative AI integration
+ vectorizer_config=Configure.Vectorizer.text2vec_cohere(), # Configure the Cohere embedding integration
+ generative_config=Configure.Generative.cohere() # Configure the Cohere generative AI integration
)
# highlight-end
# CreateCollection
diff --git a/_includes/code/python/quickstart.import_objects.py b/_includes/code/python/quickstart.import_objects.py
index be4449b618..3718cd1ab8 100644
--- a/_includes/code/python/quickstart.import_objects.py
+++ b/_includes/code/python/quickstart.import_objects.py
@@ -6,14 +6,13 @@
# Best practice: store your credentials in environment variables
wcd_url = os.environ["WCD_URL"]
wcd_api_key = os.environ["WCD_API_KEY"]
-openai_api_key = os.environ["OPENAI_APIKEY"]
+cohere_api_key = os.environ["COHERE_APIKEY"]
-# Import # ShortImport
client = weaviate.connect_to_weaviate_cloud(
cluster_url=wcd_url, # Replace with your Weaviate Cloud URL
auth_credentials=Auth.api_key(wcd_api_key), # Replace with your Weaviate Cloud key
# highlight-start
- headers={"X-OpenAI-Api-Key": openai_api_key}, # Replace with your OpenAI API key
+ headers={"X-Cohere-Api-Key": cohere_api_key}, # Replace with your Cohere API key
# highlight-end
)
@@ -33,7 +32,6 @@
"category": d["Category"],
})
# highlight-end
-# Import # END ShortImport
client.close() # Free up resources
# END Import
diff --git a/_includes/code/python/quickstart.query.neartext.py b/_includes/code/python/quickstart.query.neartext.py
index 3b083c22fa..68ea1245ba 100644
--- a/_includes/code/python/quickstart.query.neartext.py
+++ b/_includes/code/python/quickstart.query.neartext.py
@@ -6,14 +6,13 @@
# Best practice: store your credentials in environment variables
wcd_url = os.environ["WCD_URL"]
wcd_api_key = os.environ["WCD_API_KEY"]
-openai_api_key = os.environ["OPENAI_APIKEY"]
+cohere_api_key = os.environ["COHERE_APIKEY"]
-# NearText # ShortNearText
client = weaviate.connect_to_weaviate_cloud(
cluster_url=wcd_url, # Replace with your Weaviate Cloud URL
auth_credentials=Auth.api_key(wcd_api_key), # Replace with your Weaviate Cloud key
# highlight-start
- headers={"X-OpenAI-Api-Key": openai_api_key}, # Replace with your OpenAI API key
+ headers={"X-Cohere-Api-Key": cohere_api_key}, # Replace with your Cohere API key
# highlight-end
)
@@ -28,7 +27,6 @@
for obj in response.objects:
print(json.dumps(obj.properties, indent=2))
-# NearText # END ShortNearText
client.close() # Free up resources
# END NearText
diff --git a/_includes/code/python/quickstart.query.rag.py b/_includes/code/python/quickstart.query.rag.py
index c8955d7cab..b8e7b73dba 100644
--- a/_includes/code/python/quickstart.query.rag.py
+++ b/_includes/code/python/quickstart.query.rag.py
@@ -6,14 +6,13 @@
# Best practice: store your credentials in environment variables
wcd_url = os.environ["WCD_URL"]
wcd_api_key = os.environ["WCD_API_KEY"]
-openai_api_key = os.environ["OPENAI_APIKEY"]
+cohere_api_key = os.environ["COHERE_APIKEY"]
-# RAG # ShortRAG
client = weaviate.connect_to_weaviate_cloud(
cluster_url=wcd_url, # Replace with your Weaviate Cloud URL
auth_credentials=Auth.api_key(wcd_api_key), # Replace with your Weaviate Cloud key
# highlight-start
- headers={"X-OpenAI-Api-Key": openai_api_key}, # Replace with your OpenAI API key
+ headers={"X-Cohere-Api-Key": cohere_api_key}, # Replace with your Cohere API key
# highlight-end
)
@@ -28,7 +27,6 @@
# highlight-end
print(response.generated) # Inspect the generated text
-# RAG # END ShortRAG
client.close() # Free up resources
# END RAG
diff --git a/_includes/code/quickstart.autoschema.connect.docker.mdx b/_includes/code/quickstart.autoschema.connect.docker.mdx
index f9994f2533..46398dc145 100644
--- a/_includes/code/quickstart.autoschema.connect.docker.mdx
+++ b/_includes/code/quickstart.autoschema.connect.docker.mdx
@@ -21,7 +21,7 @@ import EndToEndTSCode from '!!raw-loader!/_includes/code/quickstart/endtoend.ts'
text={EndToEndTSCode}
startMarker="// DockerInstantiationExample"
endMarker="// END DockerInstantiationExample"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart.byov.all.py b/_includes/code/quickstart.byov.all.py
index 6c85a29320..7aa9e8495a 100644
--- a/_includes/code/quickstart.byov.all.py
+++ b/_includes/code/quickstart.byov.all.py
@@ -12,59 +12,60 @@
client = weaviate.connect_to_local()
-try:
+# ===== Create schema =====
+import weaviate.classes as wvc
- # ===== Create schema =====
- import weaviate.classes as wvc
+# Create the collection. Weaviate's autoschema feature will infer properties when importing.
+questions = client.collections.create(
+ "Question",
+ vectorizer_config=wvc.config.Configure.Vectorizer.none(),
+)
- # Create the collection. Weaviate's autoschema feature will infer properties when importing.
- questions = client.collections.create(
- "Question",
- vectorizer_config=wvc.config.Configure.Vectorizer.none(),
- )
+# ===== Import data =====
+import requests
- # ===== Import data =====
- import requests
+fname = "jeopardy_tiny_with_vectors_all-OpenAI-ada-002.json" # This file includes pre-generated vectors
+url = f"https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/{fname}"
+resp = requests.get(url)
+data = json.loads(resp.text) # Load data
- fname = "jeopardy_tiny_with_vectors_all-OpenAI-ada-002.json" # This file includes pre-generated vectors
- url = f"https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/{fname}"
- resp = requests.get(url)
- data = json.loads(resp.text) # Load data
+question_objs = list()
+for i, d in enumerate(data):
+ question_objs.append(wvc.data.DataObject(
+ properties={
+ "answer": d["Answer"],
+ "question": d["Question"],
+ "category": d["Category"],
+ },
+ vector=d["vector"]
+ ))
- question_objs = list()
- for i, d in enumerate(data):
- question_objs.append(wvc.data.DataObject(
- properties={
- "answer": d["Answer"],
- "question": d["Question"],
- "category": d["Category"],
- },
- vector=d["vector"]
- ))
+questions = client.collections.get("Question")
+questions.data.insert_many(question_objs)
- questions = client.collections.get("Question")
- questions.data.insert_many(question_objs) # This uses batching under the hood
+# ===== Query =====
+query_vector = [0.0042927247,-0.007413445,0.00034457954,-0.01897398,-0.00251218,0.020693844,-0.027351381,-0.008647864,-0.000042449385,-0.012337249,-0.006678342,0.0072608762,0.0051838635,-0.020555146,0.00017055604,0.028821588,0.047351733,-0.0045319796,0.008959935,-0.012323379,-0.008585449,0.022483058,0.0013869869,-0.01801696,-0.0014806085,0.01269093,0.021539906,-0.03592296,-0.0024116235,-0.012087591,0.014369184,-0.0019261781,-0.03506303,-0.028932547,-0.008162417,-0.026949156,-0.011782453,0.00049628125,0.018044699,0.011213789,0.014452403,0.028072614,-0.010957196,-0.0006033393,-0.008876716,-0.008079199,-0.011789389,-0.007517469,-0.028960286,-0.0059328363,0.006758094,0.021997612,-0.004521577,-0.015506513,-0.010208224,-0.014577232,-0.004726158,0.025714736,-0.0025104464,-0.016421925,-0.025922785,-0.00038467214,-0.021054462,0.019611996,-0.013315074,0.0014346646,-0.0017857456,0.01572843,-0.01003485,-0.008675603,0.02323203,0.0033079637,0.0056970487,-0.0129405875,0.035534605,-0.010769953,-0.023010112,-0.001464138,-0.008932196,0.015700692,0.010166614,-0.022483058,0.00788502,0.0029698857,0.018710453,0.0029768206,-0.014341445,0.008897521,-0.013141701,-0.037115768,0.0148546295,0.004067339,-0.0060680676,0.02529864,-0.027601039,0.021345729,-0.010721409,0.033676043,-0.0191959,-0.06807332,-0.009812932,0.015312335,-0.010825433,-0.018613365,-0.02110994,0.006248376,0.008550774,0.0044556954,0.023134941,-0.026990766,-0.010097264,0.021623125,-0.0070979055,-0.04798975,0.0006449489,-0.0011338618,0.0033894493,-0.011172179,0.0053641717,-0.010430141,0.021484427,-0.012885109,0.010298378,-0.013003002,0.019917132,0.010832367,-0.04235858,-0.010222093,-0.0017658077,0.009466185,0.023564907,0.01998648,0.0036928526,0.01162295,-0.022413708,0.016421925,-0.008842042,0.0043065944,-0.017503774,-0.031262685,0.007739387,0.033787,-0.040583238,-0.00029863563,0.003266354,0.025423469,0.012579971,0.023759086,0.0168935,0.0042996593,0.015437164,-0.0035125443,0.029043505,0.0031709988,-0.0038350187,-0.0030479038,-0.009889216,0.008696408,0.0022053092,0.003027099,-0.009507795,0.0028918677,0.012406598,-0.022344358,0.009695038,0.017462164,0.007059763,0.012316444,-0.0020596755,-0.019487167,-0.0132734645,0.038197618,-0.022385968,0.030153096,0.00017857457,0.019736823,0.010624319,0.0026023341,-0.014715931,0.0079405,-0.024965765,0.026200183,0.018627234,0.0108670425,-0.0010913853,0.009181853,0.01484076,-0.0072192666,0.031207206,-0.020638365,-0.004466098,0.016061308,0.0054855333,0.004379411,-0.6910524,-0.032178096,0.009660364,-0.005471663,0.012371923,0.0264221,0.007586818,-0.0030929807,-0.012434337,0.011678429,0.019320728,-0.0058392147,-0.009098634,-0.015132027,-0.013571667,-0.030014396,0.008328856,-0.03281611,0.008481425,0.002597133,-0.024008743,-0.0026543462,-0.009015415,-0.011983567,0.009154114,0.0044799675,0.018155659,-0.018821413,-0.008890586,-0.007989044,-0.0075729485,0.0119211525,-0.000356499,-0.0021550308,0.019487167,0.0044314233,-0.0039737173,0.028280662,0.021317989,0.043828785,-0.01319718,-0.030097615,0.0011035214,0.012815759,-0.022691106,0.028211314,0.016602233,-0.0020648767,0.023981003,0.015756171,0.012337249,0.0019556514,0.01583939,0.018710453,0.02334299,-0.0115536,0.017531514,-0.008946066,-0.0024619016,-0.0015447567,-0.010069525,0.0008317587,-0.014743671,-0.027295902,-0.02328751,0.027878437,-0.032483235,0.009743583,0.019917132,-0.010215159,0.022330489,0.022815935,-0.014604972,-0.0008096536,-0.0001595035,0.016075179,0.022372099,-0.0052601476,-0.015132027,0.0032004723,0.0068309107,-0.010083395,-0.02106833,-0.0066228625,0.028904807,-0.020915762,-0.007628428,0.010478686,0.0037240598,0.0037136574,0.014299835,0.02868289,0.0009223463,-0.0052428106,-0.014702061,0.009701974,-0.011810194,-0.0084952945,0.00659859,-0.013044612,-0.005218538,0.000095517884,0.021248639,0.010346922,0.011352488,0.0045909267,-0.004670678,0.020763194,0.044022962,-0.021539906,0.0077809966,-0.00095181976,-0.0062934533,0.010714474,-0.00017369844,-0.033592824,0.0066089923,-0.0025104464,0.021997612,-0.01805857,-0.0023474754,0.0060819373,0.006276116,-0.011962762,0.0061720917,0.01590874,-0.018946242,-0.020888023,-0.017670212,0.0003892232,0.017795041,0.013377489,0.011269269,-0.015464904,0.028252924,0.009681169,0.015534253,-0.020541277,0.010679799,-0.007836476,-0.027073985,-0.015561993,0.005603427,-0.0010081661,-0.013342814,-0.006154754,-0.022427578,-0.0022053092,-0.019279119,0.012933653,0.017448295,0.012815759,0.0019105745,0.028849328,0.034674674,-0.023689736,-0.0062449086,-0.0008872382,-0.0068690525,0.013516188,0.010561905,0.01814179,-0.018391447,0.003644308,-0.0026716834,-0.017032199,-0.0070181536,0.015173636,-0.0076006884,-0.04776783,0.01110283,-0.03919625,-0.015451034,0.010242898,0.011123635,0.0003929074,-0.017808912,0.01603357,0.021387339,-0.0055479477,-0.016297096,-0.0038107466,-0.020333229,-0.0032958277,0.018405316,0.0031796675,0.032316796,0.020555146,-0.004365541,0.012933653,0.009951631,0.02227501,0.0012266166,0.0085299695,0.019182028,0.009819867,0.0014468007,0.010020981,-0.0030704422,0.02314881,0.04449454,-0.00095702097,0.01794761,-0.011095895,0.0017319999,-0.042968854,-0.0022139777,-0.038197618,0.021595387,0.015437164,0.0070909704,-0.020236138,0.0027913111,-0.039473645,0.012614646,0.012316444,-0.009743583,0.001855095,-0.019681344,-0.009743583,0.00084866263,-0.017157027,0.00058080076,-0.00075634127,0.0057143862,0.017198637,0.011858738,0.012420468,-0.00026157705,-0.013849064,0.017420555,-0.0037517995,-0.001249155,0.0019539178,0.00023535434,0.02323203,0.013439903,-0.018987851,0.039751045,0.018641103,-0.0019660539,0.009119439,0.01319718,-0.0072678113,0.010464816,-0.008821237,0.016505145,0.008710277,0.0009656896,-0.0037726043,-0.006210234,-0.0020388707,-0.033620562,-0.001653115,0.01678254,-0.021359598,0.016297096,0.022857545,0.012240159,0.026935285,0.009743583,0.005721321,0.004119351,-0.0048717917,0.008821237,0.0018377576,0.00633853,-0.01159521,-0.023398468,0.008412075,-0.03287159,-0.022496928,0.0030635074,-0.01271867,0.009188788,0.021539906,-0.0035749588,0.0062969206,-0.008751887,-0.0121846795,-0.00661246,-0.0108670425,0.00954247,0.017517645,0.0061408845,-0.013509252,-0.012191615,-0.010485621,-0.00028606606,0.009965501,-0.01697672,0.0140640475,0.012538361,0.0042476472,-0.0034223902,-0.010804628,0.007732452,-0.014494013,-0.010423207,-0.0030756434,-0.0034657335,-0.0132387895,0.0005292222,-0.006678342,0.0117269745,0.008800432,-0.0062449086,-0.022427578,-0.018419186,-0.017795041,0.012288704,0.0028329208,-0.0077948663,0.0013878538,-0.0072192666,0.014604972,-0.008328856,-0.005766398,0.026852066,-0.0041505583,-0.019209769,-0.005579155,-0.02644984,-0.013682626,0.074231535,0.022982374,-0.011747779,0.04737947,0.0021376936,-0.023315249,-0.010811563,-0.0041366885,0.00527055,-0.01907107,0.006158222,-0.027060114,0.018280488,0.005957109,-0.011858738,0.0072539416,-0.014452403,-0.02220566,0.015326206,-0.0036928526,-0.017906,-0.024008743,0.0070008165,0.027129464,0.0033287685,0.0022798597,-0.0025416536,0.02231662,-0.006650602,-0.026269533,-0.003377313,0.0046360036,0.008536904,0.012462078,0.0075660134,0.019223638,0.016740931,0.0013055014,0.0001641629,-0.012545297,0.01585326,0.020929633,0.010069525,-0.018779803,0.0101042,-0.021775695,-0.0098476065,0.033481862,-0.00319007,-0.0024237595,0.019501036,0.006761561,-0.0073787705,-0.016879631,0.009403771,0.009112504,0.0047608325,0.015367815,-0.007177657,-0.00002692705,-0.027129464,-0.044217143,0.020444186,0.0017944142,0.0011052552,-0.009653429,-0.038475018,0.0013497117,-0.023592647,0.000589036,0.0035957636,-0.029820219,-0.00023080329,-0.022496928,0.008079199,-0.00632466,0.029293163,-0.011928087,0.0046810806,0.030125355,-0.0018932371,0.0013809188,-0.0026526125,-0.030069876,0.0044418257,0.0068447804,-0.0009275475,-0.0065188385,0.011401032,0.021900523,0.0061339494,0.004902999,0.02653306,0.005773333,-0.0012517556,0.00581841,0.01994487,0.020263879,-0.00839127,-0.012323379,0.0025728608,-0.023065591,-0.00661246,0.010485621,0.008210963,0.0032351469,0.0011980099,0.031290423,-0.014161136,0.007725517,0.028849328,-0.007049361,0.018155659,0.0148685,0.0041332208,0.019528776,0.0056450367,0.01801696,0.005246278,-0.0220947,-0.001432064,-0.028100355,0.013689561,0.017642474,-0.0050139576,-0.016796412,0.00212209,-0.016560623,-0.013849064,0.0007424714,-0.016089048,0.023967134,0.0030739098,-0.032677412,-0.03206714,-0.009986306,-0.019833913,0.021886652,-0.020818673,-0.011498122,0.0005365906,0.005274018,0.021914393,-0.023093332,-0.0018446926,-0.047240775,-0.01058271,-0.0022538537,-0.014923979,0.024674498,-0.013960023,-0.0022746585,-0.007864215,-0.010769953,-0.0071637873,-0.049238034,-0.022011481,-0.019958742,0.043745566,0.018641103,0.019847782,-0.016435795,0.01697672,0.016158398,0.0028953352,0.0068690525,0.004972348,0.013793585,-0.035867482,0.0191959,0.028655149,-0.006137417,0.017725693,-0.008897521,0.0023145343,0.04208118,-0.003514278,-0.020901892,-0.0026006005,-0.0135369925,-0.014098722,0.0072400714,-0.0021966405,0.014979458,-0.009244268,-0.001380052,0.006449489,-0.0015967686,-0.012108396,-0.019931002,0.03292707,-0.024521928,0.031013027,-0.017545383,-0.004667211,-0.009473121,-0.0020683443,-0.008536904,-0.002578062,0.008113873,0.0028294532,0.019736823,0.00086599996,0.0027965123,-0.0048059095,0.0063940096,-0.0034900059,-0.011525861,0.010624319,-0.014618842,0.010506426,-0.0016297096,-0.016200008,-0.03284385,0.0064078793,0.011241529,-0.0075452086,0.01693511,-0.0121846795,-0.010284508,0.038585976,-0.0045077074,0.013966958,-0.012808824,0.030985286,0.037726045,0.0015369549,-0.026796587,-0.008370466,-0.0013289069,-0.0003181401,0.02323203,0.0074689244,0.007621493,-0.027878437,0.0070181536,0.009154114,-0.012559166,-0.03706029,0.021803435,0.016297096,0.0012985665,-0.019778432,0.00033179327,-0.024951894,0.026297271,0.0065708505,-0.003065241,-0.013100091,-0.008516099,0.0007281681,0.009313617,0.0016600499,0.03270515,0.011879543,0.005801073,-0.012635451,-0.0038904983,-0.013044612,0.009459251,-0.0046845484,0.024050353,-0.01572843,0.0044834353,0.005048632,-0.0076769725,0.0003972417,-0.012469012,-0.004670678,0.03481337,-0.018322097,-0.0064182817,-0.011616015,0.02657467,0.019917132,0.010624319,-0.0038731608,-0.017725693,-0.026227923,-0.0062449086,0.022594016,0.023925524,0.0011884744,-0.012087591,-0.0014702061,-0.0063697374,-0.007857281,-0.0042615174,-0.000051063875,-0.015561993,-0.007808736,0.009944696,0.00954247,0.017808912,0.0052948226,-0.015783912,-0.0026543462,0.024660626,-0.02337073,0.027448472,-0.02117929,-0.016213877,-0.025423469,0.0069141295,-0.0012768948,-0.025173813,0.02539573,-0.017059939,-0.023551038,-0.012420468,-0.020901892,0.018211138,0.0026803522,-0.02016679,0.019667475,-0.01114444,0.017323466,-0.033287685,-0.043662347,0.023010112,-0.014396924,0.0055375453,0.0048891287,-0.011026545,0.009805998,0.014285965,0.013183311,-0.006137417,-0.024840936,-0.017184768,-0.026075354,0.020846413,-0.00029235083,-0.022066962,-0.013786649,-0.011511991,-0.012656256,0.012045981,0.0026283402,-0.0051838635,-0.0026664822,-0.008259507,0.023981003,0.011858738,-0.021623125,-0.0034657335,-0.0028936013,-0.010707539,-0.023426209,0.0075244037,-0.00037773722,0.030319534,0.0071707224,-0.0052358755,-0.018322097,0.010138874,-0.024147442,-0.018710453,0.024410969,0.005291355,0.021872783,0.0001582032,0.0069279997,0.0029109388,-0.0015733633,0.015284596,-0.026685627,-0.0052636154,0.0025173812,-0.006130482,0.02231662,-0.009133309,-0.010978001,-0.023981003,0.021581516,-0.007063231,0.0048925965,0.00223825,-0.004847519,0.0050659697,-0.0082664415,0.023634257,0.010430141,0.014244355,0.015367815,-0.019501036,-0.009930826,0.013301204,-0.015340075,-0.024757717,-0.008065329,-0.016338706,-0.008751887,0.036366798,-0.011574405,-0.0129405875,0.022108572,-0.010326117,0.015783912,0.018821413,-0.0024358958,0.0066401996,0.0016513813,-0.007878086,-0.014369184,-0.0057109185,-0.027642649,-0.0047296253,0.014729801,-0.00065751845,0.020763194,-0.01210146,0.0009795595,-0.013349749,-0.0024653692,-0.024258401,-0.0013471111,-0.0051630586,0.015298465,0.021650866,-0.02314881,-0.02427227,0.0000668571,-0.018322097,-0.007933565,-0.011498122,0.010908652,-0.0074758595,0.0090570245,0.005100644,-0.018419186,-0.014383054,-0.0051457216,0.012788019,-0.022843674,-0.010138874,0.17021103,-0.031345904,0.0020614092,0.024091963,-0.004341269,0.005180396,0.019182028,-0.0011312612,0.0013644483,0.016408054,0.0028173171,-0.0062379735,-0.01106122,0.00850223,0.008897521,-0.0047747022,-0.02334299,-0.012801889,-0.0202084,-0.014181941,0.0033305022,-0.0015196175,-0.0083011165,-0.03167878,0.031123986,0.0016505144,-0.01003485,-0.021997612,0.004341269,-0.0058946945,-0.011241529,-0.016477404,-0.00028324872,-0.009785193,-0.019639734,0.0076561677,-0.0029525484,0.0037275273,0.016477404,0.00036668466,0.0036720478,0.010783823,0.029209943,-0.027573299,0.0009691571,0.034646932,-0.0055652848,-0.017906,0.009133309,0.01592261,-0.031651042,0.01803083,-0.0057906704,0.032400016,0.0072955512,-0.0072539416,0.013030742,0.0019747226,0.006775431,0.023564907,-0.027143333,0.01586713,-0.018585624,0.018114049,-0.01059658,0.010360792,-0.011310878,0.010125005,0.0028398556,-0.015312335,-0.015104287,-0.0025936656,-0.022066962,0.010395466,-0.028128095,-0.019889392,0.041581865,0.0356733,0.0190572,0.010055655,0.000041853415,0.0025919317,-0.018918501,-0.022372099,-0.010749148,-0.027198814,0.015284596,0.020263879,-0.014937849,0.01803083,-0.019556515,-0.0023925523,-0.020569015,-0.024008743,0.010201288,0.018086309,-0.006560448,0.030402754,-0.029154465,0.0106381895,-0.028627409,0.016768672,0.03181748,0.0007095305,-0.0006332462,0.0031467266,-0.027184943,0.0055340775,0.0022573213,-0.018710453,0.0006384474,-0.017378947,-0.005128384,-0.008398206,0.0022642561,0.019806173,0.0023682802,-0.016283225,0.0190572,-0.013190245,-0.015007198,-0.010229029,0.0122193545,0.011331683,-0.008516099,-0.007989044,-0.014188876,0.018238878,0.009043154,-0.040444538,-0.011213789,-0.0019383142,0.0014476676,-0.006449489,-0.017254118,-0.0006939269,-0.018835282,-0.013703431,-0.02434162,0.011747779,-0.0025364524,0.0007021621,-0.0036339057,0.007489729,-0.0034848046,-0.014951719,0.03167878,0.012753344,-0.005176929,-0.022732716,-0.016505145,-0.005097177,-0.026255662,-0.008842042,0.010887847,-0.000096547294,-0.00951473,-0.03481337,0.0070146862,0.007975175,-0.022427578,0.0053086923,0.03511851,-0.012302574,0.013335879,-0.018654974,-0.18252748,0.0028346544,0.024577407,-0.029709259,0.027864566,0.025797956,0.002408156,-0.0002698123,0.005759463,0.0030877795,0.012801889,0.0074550547,-0.04152639,-0.0018689649,-0.012857368,-0.0084952945,-0.000054964774,0.011331683,0.0029178737,0.010804628,0.023828436,-0.020693844,0.011498122,-0.022774326,0.007115243,0.018405316,0.0005461261,0.009327487,-0.0040846765,-0.028544191,-0.010686734,-0.018488536,0.017281856,-0.0012309508,-0.0150488075,0.014799151,-0.011872608,-0.027711999,-0.01708768,0.008321921,0.0059120315,0.040860634,0.013446838,-0.0037483322,0.005152656,0.012073721,0.009001545,0.012788019,0.027351381,-0.026366621,0.0042337775,-0.0018013492,0.0030756434,0.005430054,0.033315424,0.0012734274,-0.043939743,0.025950525,-0.02099898,-0.006151287,-0.021012852,-0.028433232,0.020721585,-0.016879631,-0.010520295,-0.0135369925,-0.024258401,0.0103746625,-0.019931002,0.003027099,0.008204027,-0.013176376,0.024716107,-0.012919783,0.0058808243,-0.0012612912,0.0066540698,0.010811563,-0.0015248187,-0.0140640475,-0.02228888,0.025978265,-0.012489817,-0.018807542,0.0085646445,-0.0014832091,-0.0016305764,0.0018308227,-0.007056296,-0.016019698,0.0046186666,-0.024091963,-0.018738193,-0.021470558,0.01482689,0.027947785,-0.007850346,0.012288704,-0.0026976895,-0.014563362,0.014175006,-0.00047027523,-0.0048856614,0.011505056,0.04760139,-0.0046498734,0.019667475,-0.008536904,0.027975526,0.0023006645,-0.009570209,-0.0010359058,0.012018242,0.035867482,-0.0077671264,0.007760192,-0.012510622,-0.01159521,0.0038731608,-0.009951631,0.040943854,-0.0036616453,-0.0190572,0.0127464095,0.0006341131,-0.015353945,-0.10768566,-0.02848871,-0.00011756881,0.02644984,-0.011935023,0.021942133,-0.031207206,0.014674322,0.0074273148,0.031872958,-0.019445557,-0.002134226,-0.01893237,-0.010097264,0.0011399299,0.007933565,-0.0043308665,-0.009882282,-0.0069349343,0.036810633,0.005305225,-0.0076492326,0.006054198,0.014563362,-0.011401032,0.011851803,-0.0024896415,0.03706029,0.008349661,-0.0019209769,0.002571127,-0.019778432,0.007760192,-0.032316796,-0.008356596,0.0006618528,-0.014618842,-0.014341445,0.016047439,-0.0033374373,0.02314881,0.007205397,0.021207029,-0.017739562,0.011435707,-0.013710366,0.0012274834,0.004719223,-0.0022642561,-0.026963025,-0.010360792,0.00084172765,-0.018474666,-0.00082742434,0.0093899015,-0.013107026,-0.0028727967,0.02002809,-0.009632624,-0.0014754073,0.010353858,0.0016383782,-0.0068933247,0.04435584,0.008363531,-0.015742302,-0.02321816,-0.026130833,0.027850697,-0.023509428,0.016019698,0.03256645,-0.025936656,0.011151374,-0.019306857,0.0129405875,-0.016671583,0.010548036,0.02013905,-0.022815935,-0.009126374,-0.019140419,0.016130658,-0.020804804,0.026644018,0.011588275,-0.005128384,-0.006664472,0.008086134,0.002212244,-0.0153816845,0.008294182,0.021415077,-0.009854542,-0.0032299457,-0.0010731812,-0.01482689,-0.027309772,0.00066098594,0.010450946,-0.012316444,-0.007871151,-0.072456196,0.0009197457,0.008204027,-0.005371107,0.017628603,0.0020267346,0.005381509,-0.0079405,0.009896152,0.02965378,-0.015298465,0.018322097,-0.0013601141,-0.02432775,-0.019140419,-0.028294533,0.035534605,0.02016679,0.020041961,0.015312335,0.022663366,-0.0050798394,0.037975702,0.0039355755,-0.023856174,-0.007822606,-0.0023162682,0.010215159,-0.013897609,-0.028031005,0.01055497,-0.011338618,-0.00012883807,0.026865937,0.0074550547,-0.017434426,0.014077917,0.00034067867,-0.00013642316,0.06025071,-0.02525703,-0.03620036,-0.0070909704,-0.009438446,0.01215694,0.01800309,-0.027004635,0.02009744,0.0064529567,-0.01316944,0.0061998316,0.013550862,-0.02119316,-0.034646932,-0.012045981,-0.010409337,0.019681344,0.0080029145,0.011935023,-0.0077879312,0.028155833,0.048433583,0.0026526125,0.00042476473,-0.014168072,-0.00633853,-0.009230398,-0.008162417,0.012371923,-0.030957548,-0.0200697,-0.00948699,0.0026335414,0.005031295,0.019098809,-0.008308051,-0.02319042,0.008959935,-0.0030635074,0.028627409,0.025548298,0.016269356,-0.016574493,0.026741108,0.023634257,0.024993503,-0.01678254,0.009029285,0.012045981,0.0071637873,-0.007961305,0.015756171,0.0061270148,-0.003606166,0.026810456,0.014660452,-0.005825345,-0.014230486,0.005142254,0.0134676425,-0.014660452,-0.007080568,0.0054508587,-0.020555146,-0.0020926164,-0.0039667827,-0.026158573,-0.042663716,-0.009570209,0.013980828,0.019681344,0.004802442,-0.010721409,0.006702614,-0.025492819,0.016089048,-0.006990414,-0.01798922,-0.018169528,0.037753783,0.021359598,0.011775519,0.023634257,0.010936392,0.00850223,0.01214307,0.028877066,-0.0057698656,0.0021671671,0.0044140858,0.012801889,-0.008807367,-0.029293163,-0.010229029,-0.0018134854,-0.023842305,0.0101458095,0.017739562,-0.029431863,0.047934268,0.0026266065,-0.004015327,0.016089048,-0.02205309,0.012337249,0.014244355,0.020360967,-0.04260824,-0.010797693,0.009070895,-0.0006332462,0.0036789828,-0.0140293725,-0.032122616,-0.0036893853,-0.01904333,0.019265248,0.002271191,-0.0055132727,-0.0018464263,-0.0055028703,0.029764738,0.009327487,-0.02868289,-0.0085646445,0.014168072,-0.0011243263,-0.0057109185,-0.024022613,0.028932547,0.0028450568,-0.04629762,-0.011338618,0.0070250886,-0.00013393092,-0.021720216,-0.007961305,0.012801889,0.028072614,-0.0042407126,0.018682713,-0.019722953,-0.022538537,-0.0032507505,0.010645124,-0.021803435,-0.020763194,-0.025992135]
- # ===== Query =====
- query_vector = [0.0042927247,-0.007413445,0.00034457954,-0.01897398,-0.00251218,0.020693844,-0.027351381,-0.008647864,-0.000042449385,-0.012337249,-0.006678342,0.0072608762,0.0051838635,-0.020555146,0.00017055604,0.028821588,0.047351733,-0.0045319796,0.008959935,-0.012323379,-0.008585449,0.022483058,0.0013869869,-0.01801696,-0.0014806085,0.01269093,0.021539906,-0.03592296,-0.0024116235,-0.012087591,0.014369184,-0.0019261781,-0.03506303,-0.028932547,-0.008162417,-0.026949156,-0.011782453,0.00049628125,0.018044699,0.011213789,0.014452403,0.028072614,-0.010957196,-0.0006033393,-0.008876716,-0.008079199,-0.011789389,-0.007517469,-0.028960286,-0.0059328363,0.006758094,0.021997612,-0.004521577,-0.015506513,-0.010208224,-0.014577232,-0.004726158,0.025714736,-0.0025104464,-0.016421925,-0.025922785,-0.00038467214,-0.021054462,0.019611996,-0.013315074,0.0014346646,-0.0017857456,0.01572843,-0.01003485,-0.008675603,0.02323203,0.0033079637,0.0056970487,-0.0129405875,0.035534605,-0.010769953,-0.023010112,-0.001464138,-0.008932196,0.015700692,0.010166614,-0.022483058,0.00788502,0.0029698857,0.018710453,0.0029768206,-0.014341445,0.008897521,-0.013141701,-0.037115768,0.0148546295,0.004067339,-0.0060680676,0.02529864,-0.027601039,0.021345729,-0.010721409,0.033676043,-0.0191959,-0.06807332,-0.009812932,0.015312335,-0.010825433,-0.018613365,-0.02110994,0.006248376,0.008550774,0.0044556954,0.023134941,-0.026990766,-0.010097264,0.021623125,-0.0070979055,-0.04798975,0.0006449489,-0.0011338618,0.0033894493,-0.011172179,0.0053641717,-0.010430141,0.021484427,-0.012885109,0.010298378,-0.013003002,0.019917132,0.010832367,-0.04235858,-0.010222093,-0.0017658077,0.009466185,0.023564907,0.01998648,0.0036928526,0.01162295,-0.022413708,0.016421925,-0.008842042,0.0043065944,-0.017503774,-0.031262685,0.007739387,0.033787,-0.040583238,-0.00029863563,0.003266354,0.025423469,0.012579971,0.023759086,0.0168935,0.0042996593,0.015437164,-0.0035125443,0.029043505,0.0031709988,-0.0038350187,-0.0030479038,-0.009889216,0.008696408,0.0022053092,0.003027099,-0.009507795,0.0028918677,0.012406598,-0.022344358,0.009695038,0.017462164,0.007059763,0.012316444,-0.0020596755,-0.019487167,-0.0132734645,0.038197618,-0.022385968,0.030153096,0.00017857457,0.019736823,0.010624319,0.0026023341,-0.014715931,0.0079405,-0.024965765,0.026200183,0.018627234,0.0108670425,-0.0010913853,0.009181853,0.01484076,-0.0072192666,0.031207206,-0.020638365,-0.004466098,0.016061308,0.0054855333,0.004379411,-0.6910524,-0.032178096,0.009660364,-0.005471663,0.012371923,0.0264221,0.007586818,-0.0030929807,-0.012434337,0.011678429,0.019320728,-0.0058392147,-0.009098634,-0.015132027,-0.013571667,-0.030014396,0.008328856,-0.03281611,0.008481425,0.002597133,-0.024008743,-0.0026543462,-0.009015415,-0.011983567,0.009154114,0.0044799675,0.018155659,-0.018821413,-0.008890586,-0.007989044,-0.0075729485,0.0119211525,-0.000356499,-0.0021550308,0.019487167,0.0044314233,-0.0039737173,0.028280662,0.021317989,0.043828785,-0.01319718,-0.030097615,0.0011035214,0.012815759,-0.022691106,0.028211314,0.016602233,-0.0020648767,0.023981003,0.015756171,0.012337249,0.0019556514,0.01583939,0.018710453,0.02334299,-0.0115536,0.017531514,-0.008946066,-0.0024619016,-0.0015447567,-0.010069525,0.0008317587,-0.014743671,-0.027295902,-0.02328751,0.027878437,-0.032483235,0.009743583,0.019917132,-0.010215159,0.022330489,0.022815935,-0.014604972,-0.0008096536,-0.0001595035,0.016075179,0.022372099,-0.0052601476,-0.015132027,0.0032004723,0.0068309107,-0.010083395,-0.02106833,-0.0066228625,0.028904807,-0.020915762,-0.007628428,0.010478686,0.0037240598,0.0037136574,0.014299835,0.02868289,0.0009223463,-0.0052428106,-0.014702061,0.009701974,-0.011810194,-0.0084952945,0.00659859,-0.013044612,-0.005218538,0.000095517884,0.021248639,0.010346922,0.011352488,0.0045909267,-0.004670678,0.020763194,0.044022962,-0.021539906,0.0077809966,-0.00095181976,-0.0062934533,0.010714474,-0.00017369844,-0.033592824,0.0066089923,-0.0025104464,0.021997612,-0.01805857,-0.0023474754,0.0060819373,0.006276116,-0.011962762,0.0061720917,0.01590874,-0.018946242,-0.020888023,-0.017670212,0.0003892232,0.017795041,0.013377489,0.011269269,-0.015464904,0.028252924,0.009681169,0.015534253,-0.020541277,0.010679799,-0.007836476,-0.027073985,-0.015561993,0.005603427,-0.0010081661,-0.013342814,-0.006154754,-0.022427578,-0.0022053092,-0.019279119,0.012933653,0.017448295,0.012815759,0.0019105745,0.028849328,0.034674674,-0.023689736,-0.0062449086,-0.0008872382,-0.0068690525,0.013516188,0.010561905,0.01814179,-0.018391447,0.003644308,-0.0026716834,-0.017032199,-0.0070181536,0.015173636,-0.0076006884,-0.04776783,0.01110283,-0.03919625,-0.015451034,0.010242898,0.011123635,0.0003929074,-0.017808912,0.01603357,0.021387339,-0.0055479477,-0.016297096,-0.0038107466,-0.020333229,-0.0032958277,0.018405316,0.0031796675,0.032316796,0.020555146,-0.004365541,0.012933653,0.009951631,0.02227501,0.0012266166,0.0085299695,0.019182028,0.009819867,0.0014468007,0.010020981,-0.0030704422,0.02314881,0.04449454,-0.00095702097,0.01794761,-0.011095895,0.0017319999,-0.042968854,-0.0022139777,-0.038197618,0.021595387,0.015437164,0.0070909704,-0.020236138,0.0027913111,-0.039473645,0.012614646,0.012316444,-0.009743583,0.001855095,-0.019681344,-0.009743583,0.00084866263,-0.017157027,0.00058080076,-0.00075634127,0.0057143862,0.017198637,0.011858738,0.012420468,-0.00026157705,-0.013849064,0.017420555,-0.0037517995,-0.001249155,0.0019539178,0.00023535434,0.02323203,0.013439903,-0.018987851,0.039751045,0.018641103,-0.0019660539,0.009119439,0.01319718,-0.0072678113,0.010464816,-0.008821237,0.016505145,0.008710277,0.0009656896,-0.0037726043,-0.006210234,-0.0020388707,-0.033620562,-0.001653115,0.01678254,-0.021359598,0.016297096,0.022857545,0.012240159,0.026935285,0.009743583,0.005721321,0.004119351,-0.0048717917,0.008821237,0.0018377576,0.00633853,-0.01159521,-0.023398468,0.008412075,-0.03287159,-0.022496928,0.0030635074,-0.01271867,0.009188788,0.021539906,-0.0035749588,0.0062969206,-0.008751887,-0.0121846795,-0.00661246,-0.0108670425,0.00954247,0.017517645,0.0061408845,-0.013509252,-0.012191615,-0.010485621,-0.00028606606,0.009965501,-0.01697672,0.0140640475,0.012538361,0.0042476472,-0.0034223902,-0.010804628,0.007732452,-0.014494013,-0.010423207,-0.0030756434,-0.0034657335,-0.0132387895,0.0005292222,-0.006678342,0.0117269745,0.008800432,-0.0062449086,-0.022427578,-0.018419186,-0.017795041,0.012288704,0.0028329208,-0.0077948663,0.0013878538,-0.0072192666,0.014604972,-0.008328856,-0.005766398,0.026852066,-0.0041505583,-0.019209769,-0.005579155,-0.02644984,-0.013682626,0.074231535,0.022982374,-0.011747779,0.04737947,0.0021376936,-0.023315249,-0.010811563,-0.0041366885,0.00527055,-0.01907107,0.006158222,-0.027060114,0.018280488,0.005957109,-0.011858738,0.0072539416,-0.014452403,-0.02220566,0.015326206,-0.0036928526,-0.017906,-0.024008743,0.0070008165,0.027129464,0.0033287685,0.0022798597,-0.0025416536,0.02231662,-0.006650602,-0.026269533,-0.003377313,0.0046360036,0.008536904,0.012462078,0.0075660134,0.019223638,0.016740931,0.0013055014,0.0001641629,-0.012545297,0.01585326,0.020929633,0.010069525,-0.018779803,0.0101042,-0.021775695,-0.0098476065,0.033481862,-0.00319007,-0.0024237595,0.019501036,0.006761561,-0.0073787705,-0.016879631,0.009403771,0.009112504,0.0047608325,0.015367815,-0.007177657,-0.00002692705,-0.027129464,-0.044217143,0.020444186,0.0017944142,0.0011052552,-0.009653429,-0.038475018,0.0013497117,-0.023592647,0.000589036,0.0035957636,-0.029820219,-0.00023080329,-0.022496928,0.008079199,-0.00632466,0.029293163,-0.011928087,0.0046810806,0.030125355,-0.0018932371,0.0013809188,-0.0026526125,-0.030069876,0.0044418257,0.0068447804,-0.0009275475,-0.0065188385,0.011401032,0.021900523,0.0061339494,0.004902999,0.02653306,0.005773333,-0.0012517556,0.00581841,0.01994487,0.020263879,-0.00839127,-0.012323379,0.0025728608,-0.023065591,-0.00661246,0.010485621,0.008210963,0.0032351469,0.0011980099,0.031290423,-0.014161136,0.007725517,0.028849328,-0.007049361,0.018155659,0.0148685,0.0041332208,0.019528776,0.0056450367,0.01801696,0.005246278,-0.0220947,-0.001432064,-0.028100355,0.013689561,0.017642474,-0.0050139576,-0.016796412,0.00212209,-0.016560623,-0.013849064,0.0007424714,-0.016089048,0.023967134,0.0030739098,-0.032677412,-0.03206714,-0.009986306,-0.019833913,0.021886652,-0.020818673,-0.011498122,0.0005365906,0.005274018,0.021914393,-0.023093332,-0.0018446926,-0.047240775,-0.01058271,-0.0022538537,-0.014923979,0.024674498,-0.013960023,-0.0022746585,-0.007864215,-0.010769953,-0.0071637873,-0.049238034,-0.022011481,-0.019958742,0.043745566,0.018641103,0.019847782,-0.016435795,0.01697672,0.016158398,0.0028953352,0.0068690525,0.004972348,0.013793585,-0.035867482,0.0191959,0.028655149,-0.006137417,0.017725693,-0.008897521,0.0023145343,0.04208118,-0.003514278,-0.020901892,-0.0026006005,-0.0135369925,-0.014098722,0.0072400714,-0.0021966405,0.014979458,-0.009244268,-0.001380052,0.006449489,-0.0015967686,-0.012108396,-0.019931002,0.03292707,-0.024521928,0.031013027,-0.017545383,-0.004667211,-0.009473121,-0.0020683443,-0.008536904,-0.002578062,0.008113873,0.0028294532,0.019736823,0.00086599996,0.0027965123,-0.0048059095,0.0063940096,-0.0034900059,-0.011525861,0.010624319,-0.014618842,0.010506426,-0.0016297096,-0.016200008,-0.03284385,0.0064078793,0.011241529,-0.0075452086,0.01693511,-0.0121846795,-0.010284508,0.038585976,-0.0045077074,0.013966958,-0.012808824,0.030985286,0.037726045,0.0015369549,-0.026796587,-0.008370466,-0.0013289069,-0.0003181401,0.02323203,0.0074689244,0.007621493,-0.027878437,0.0070181536,0.009154114,-0.012559166,-0.03706029,0.021803435,0.016297096,0.0012985665,-0.019778432,0.00033179327,-0.024951894,0.026297271,0.0065708505,-0.003065241,-0.013100091,-0.008516099,0.0007281681,0.009313617,0.0016600499,0.03270515,0.011879543,0.005801073,-0.012635451,-0.0038904983,-0.013044612,0.009459251,-0.0046845484,0.024050353,-0.01572843,0.0044834353,0.005048632,-0.0076769725,0.0003972417,-0.012469012,-0.004670678,0.03481337,-0.018322097,-0.0064182817,-0.011616015,0.02657467,0.019917132,0.010624319,-0.0038731608,-0.017725693,-0.026227923,-0.0062449086,0.022594016,0.023925524,0.0011884744,-0.012087591,-0.0014702061,-0.0063697374,-0.007857281,-0.0042615174,-0.000051063875,-0.015561993,-0.007808736,0.009944696,0.00954247,0.017808912,0.0052948226,-0.015783912,-0.0026543462,0.024660626,-0.02337073,0.027448472,-0.02117929,-0.016213877,-0.025423469,0.0069141295,-0.0012768948,-0.025173813,0.02539573,-0.017059939,-0.023551038,-0.012420468,-0.020901892,0.018211138,0.0026803522,-0.02016679,0.019667475,-0.01114444,0.017323466,-0.033287685,-0.043662347,0.023010112,-0.014396924,0.0055375453,0.0048891287,-0.011026545,0.009805998,0.014285965,0.013183311,-0.006137417,-0.024840936,-0.017184768,-0.026075354,0.020846413,-0.00029235083,-0.022066962,-0.013786649,-0.011511991,-0.012656256,0.012045981,0.0026283402,-0.0051838635,-0.0026664822,-0.008259507,0.023981003,0.011858738,-0.021623125,-0.0034657335,-0.0028936013,-0.010707539,-0.023426209,0.0075244037,-0.00037773722,0.030319534,0.0071707224,-0.0052358755,-0.018322097,0.010138874,-0.024147442,-0.018710453,0.024410969,0.005291355,0.021872783,0.0001582032,0.0069279997,0.0029109388,-0.0015733633,0.015284596,-0.026685627,-0.0052636154,0.0025173812,-0.006130482,0.02231662,-0.009133309,-0.010978001,-0.023981003,0.021581516,-0.007063231,0.0048925965,0.00223825,-0.004847519,0.0050659697,-0.0082664415,0.023634257,0.010430141,0.014244355,0.015367815,-0.019501036,-0.009930826,0.013301204,-0.015340075,-0.024757717,-0.008065329,-0.016338706,-0.008751887,0.036366798,-0.011574405,-0.0129405875,0.022108572,-0.010326117,0.015783912,0.018821413,-0.0024358958,0.0066401996,0.0016513813,-0.007878086,-0.014369184,-0.0057109185,-0.027642649,-0.0047296253,0.014729801,-0.00065751845,0.020763194,-0.01210146,0.0009795595,-0.013349749,-0.0024653692,-0.024258401,-0.0013471111,-0.0051630586,0.015298465,0.021650866,-0.02314881,-0.02427227,0.0000668571,-0.018322097,-0.007933565,-0.011498122,0.010908652,-0.0074758595,0.0090570245,0.005100644,-0.018419186,-0.014383054,-0.0051457216,0.012788019,-0.022843674,-0.010138874,0.17021103,-0.031345904,0.0020614092,0.024091963,-0.004341269,0.005180396,0.019182028,-0.0011312612,0.0013644483,0.016408054,0.0028173171,-0.0062379735,-0.01106122,0.00850223,0.008897521,-0.0047747022,-0.02334299,-0.012801889,-0.0202084,-0.014181941,0.0033305022,-0.0015196175,-0.0083011165,-0.03167878,0.031123986,0.0016505144,-0.01003485,-0.021997612,0.004341269,-0.0058946945,-0.011241529,-0.016477404,-0.00028324872,-0.009785193,-0.019639734,0.0076561677,-0.0029525484,0.0037275273,0.016477404,0.00036668466,0.0036720478,0.010783823,0.029209943,-0.027573299,0.0009691571,0.034646932,-0.0055652848,-0.017906,0.009133309,0.01592261,-0.031651042,0.01803083,-0.0057906704,0.032400016,0.0072955512,-0.0072539416,0.013030742,0.0019747226,0.006775431,0.023564907,-0.027143333,0.01586713,-0.018585624,0.018114049,-0.01059658,0.010360792,-0.011310878,0.010125005,0.0028398556,-0.015312335,-0.015104287,-0.0025936656,-0.022066962,0.010395466,-0.028128095,-0.019889392,0.041581865,0.0356733,0.0190572,0.010055655,0.000041853415,0.0025919317,-0.018918501,-0.022372099,-0.010749148,-0.027198814,0.015284596,0.020263879,-0.014937849,0.01803083,-0.019556515,-0.0023925523,-0.020569015,-0.024008743,0.010201288,0.018086309,-0.006560448,0.030402754,-0.029154465,0.0106381895,-0.028627409,0.016768672,0.03181748,0.0007095305,-0.0006332462,0.0031467266,-0.027184943,0.0055340775,0.0022573213,-0.018710453,0.0006384474,-0.017378947,-0.005128384,-0.008398206,0.0022642561,0.019806173,0.0023682802,-0.016283225,0.0190572,-0.013190245,-0.015007198,-0.010229029,0.0122193545,0.011331683,-0.008516099,-0.007989044,-0.014188876,0.018238878,0.009043154,-0.040444538,-0.011213789,-0.0019383142,0.0014476676,-0.006449489,-0.017254118,-0.0006939269,-0.018835282,-0.013703431,-0.02434162,0.011747779,-0.0025364524,0.0007021621,-0.0036339057,0.007489729,-0.0034848046,-0.014951719,0.03167878,0.012753344,-0.005176929,-0.022732716,-0.016505145,-0.005097177,-0.026255662,-0.008842042,0.010887847,-0.000096547294,-0.00951473,-0.03481337,0.0070146862,0.007975175,-0.022427578,0.0053086923,0.03511851,-0.012302574,0.013335879,-0.018654974,-0.18252748,0.0028346544,0.024577407,-0.029709259,0.027864566,0.025797956,0.002408156,-0.0002698123,0.005759463,0.0030877795,0.012801889,0.0074550547,-0.04152639,-0.0018689649,-0.012857368,-0.0084952945,-0.000054964774,0.011331683,0.0029178737,0.010804628,0.023828436,-0.020693844,0.011498122,-0.022774326,0.007115243,0.018405316,0.0005461261,0.009327487,-0.0040846765,-0.028544191,-0.010686734,-0.018488536,0.017281856,-0.0012309508,-0.0150488075,0.014799151,-0.011872608,-0.027711999,-0.01708768,0.008321921,0.0059120315,0.040860634,0.013446838,-0.0037483322,0.005152656,0.012073721,0.009001545,0.012788019,0.027351381,-0.026366621,0.0042337775,-0.0018013492,0.0030756434,0.005430054,0.033315424,0.0012734274,-0.043939743,0.025950525,-0.02099898,-0.006151287,-0.021012852,-0.028433232,0.020721585,-0.016879631,-0.010520295,-0.0135369925,-0.024258401,0.0103746625,-0.019931002,0.003027099,0.008204027,-0.013176376,0.024716107,-0.012919783,0.0058808243,-0.0012612912,0.0066540698,0.010811563,-0.0015248187,-0.0140640475,-0.02228888,0.025978265,-0.012489817,-0.018807542,0.0085646445,-0.0014832091,-0.0016305764,0.0018308227,-0.007056296,-0.016019698,0.0046186666,-0.024091963,-0.018738193,-0.021470558,0.01482689,0.027947785,-0.007850346,0.012288704,-0.0026976895,-0.014563362,0.014175006,-0.00047027523,-0.0048856614,0.011505056,0.04760139,-0.0046498734,0.019667475,-0.008536904,0.027975526,0.0023006645,-0.009570209,-0.0010359058,0.012018242,0.035867482,-0.0077671264,0.007760192,-0.012510622,-0.01159521,0.0038731608,-0.009951631,0.040943854,-0.0036616453,-0.0190572,0.0127464095,0.0006341131,-0.015353945,-0.10768566,-0.02848871,-0.00011756881,0.02644984,-0.011935023,0.021942133,-0.031207206,0.014674322,0.0074273148,0.031872958,-0.019445557,-0.002134226,-0.01893237,-0.010097264,0.0011399299,0.007933565,-0.0043308665,-0.009882282,-0.0069349343,0.036810633,0.005305225,-0.0076492326,0.006054198,0.014563362,-0.011401032,0.011851803,-0.0024896415,0.03706029,0.008349661,-0.0019209769,0.002571127,-0.019778432,0.007760192,-0.032316796,-0.008356596,0.0006618528,-0.014618842,-0.014341445,0.016047439,-0.0033374373,0.02314881,0.007205397,0.021207029,-0.017739562,0.011435707,-0.013710366,0.0012274834,0.004719223,-0.0022642561,-0.026963025,-0.010360792,0.00084172765,-0.018474666,-0.00082742434,0.0093899015,-0.013107026,-0.0028727967,0.02002809,-0.009632624,-0.0014754073,0.010353858,0.0016383782,-0.0068933247,0.04435584,0.008363531,-0.015742302,-0.02321816,-0.026130833,0.027850697,-0.023509428,0.016019698,0.03256645,-0.025936656,0.011151374,-0.019306857,0.0129405875,-0.016671583,0.010548036,0.02013905,-0.022815935,-0.009126374,-0.019140419,0.016130658,-0.020804804,0.026644018,0.011588275,-0.005128384,-0.006664472,0.008086134,0.002212244,-0.0153816845,0.008294182,0.021415077,-0.009854542,-0.0032299457,-0.0010731812,-0.01482689,-0.027309772,0.00066098594,0.010450946,-0.012316444,-0.007871151,-0.072456196,0.0009197457,0.008204027,-0.005371107,0.017628603,0.0020267346,0.005381509,-0.0079405,0.009896152,0.02965378,-0.015298465,0.018322097,-0.0013601141,-0.02432775,-0.019140419,-0.028294533,0.035534605,0.02016679,0.020041961,0.015312335,0.022663366,-0.0050798394,0.037975702,0.0039355755,-0.023856174,-0.007822606,-0.0023162682,0.010215159,-0.013897609,-0.028031005,0.01055497,-0.011338618,-0.00012883807,0.026865937,0.0074550547,-0.017434426,0.014077917,0.00034067867,-0.00013642316,0.06025071,-0.02525703,-0.03620036,-0.0070909704,-0.009438446,0.01215694,0.01800309,-0.027004635,0.02009744,0.0064529567,-0.01316944,0.0061998316,0.013550862,-0.02119316,-0.034646932,-0.012045981,-0.010409337,0.019681344,0.0080029145,0.011935023,-0.0077879312,0.028155833,0.048433583,0.0026526125,0.00042476473,-0.014168072,-0.00633853,-0.009230398,-0.008162417,0.012371923,-0.030957548,-0.0200697,-0.00948699,0.0026335414,0.005031295,0.019098809,-0.008308051,-0.02319042,0.008959935,-0.0030635074,0.028627409,0.025548298,0.016269356,-0.016574493,0.026741108,0.023634257,0.024993503,-0.01678254,0.009029285,0.012045981,0.0071637873,-0.007961305,0.015756171,0.0061270148,-0.003606166,0.026810456,0.014660452,-0.005825345,-0.014230486,0.005142254,0.0134676425,-0.014660452,-0.007080568,0.0054508587,-0.020555146,-0.0020926164,-0.0039667827,-0.026158573,-0.042663716,-0.009570209,0.013980828,0.019681344,0.004802442,-0.010721409,0.006702614,-0.025492819,0.016089048,-0.006990414,-0.01798922,-0.018169528,0.037753783,0.021359598,0.011775519,0.023634257,0.010936392,0.00850223,0.01214307,0.028877066,-0.0057698656,0.0021671671,0.0044140858,0.012801889,-0.008807367,-0.029293163,-0.010229029,-0.0018134854,-0.023842305,0.0101458095,0.017739562,-0.029431863,0.047934268,0.0026266065,-0.004015327,0.016089048,-0.02205309,0.012337249,0.014244355,0.020360967,-0.04260824,-0.010797693,0.009070895,-0.0006332462,0.0036789828,-0.0140293725,-0.032122616,-0.0036893853,-0.01904333,0.019265248,0.002271191,-0.0055132727,-0.0018464263,-0.0055028703,0.029764738,0.009327487,-0.02868289,-0.0085646445,0.014168072,-0.0011243263,-0.0057109185,-0.024022613,0.028932547,0.0028450568,-0.04629762,-0.011338618,0.0070250886,-0.00013393092,-0.021720216,-0.007961305,0.012801889,0.028072614,-0.0042407126,0.018682713,-0.019722953,-0.022538537,-0.0032507505,0.010645124,-0.021803435,-0.020763194,-0.025992135]
+import time
+time.sleep(1) # Sleep so we don't query before async indexing finishes
- response = questions.query.near_vector(
- near_vector=query_vector,
- limit=2,
- return_metadata=wvc.query.MetadataQuery(certainty=True)
- )
+response = questions.query.near_vector(
+ near_vector=query_vector,
+ limit=2,
+ return_metadata=wvc.query.MetadataQuery(certainty=True)
+)
- print(response)
+print(response)
- # ===== Test import =====
- # ===== Test query results =====
- assert len(response.objects) == 2
- assert type(response.objects[0].metadata.certainty) == float
- assert type(response.objects[0].properties) == dict
+# ===== Test import =====
- client.collections.delete("Question")
+# ===== Test query results =====
+assert len(response.objects) == 2
+assert type(response.objects[0].metadata.certainty) == float
+assert type(response.objects[0].properties) == dict
+
+client.collections.delete("Question")
-finally:
- client.close()
+client.close()
diff --git a/_includes/code/quickstart.byov.all.ts b/_includes/code/quickstart.byov.all.ts
index ed07420c9d..1f53109d66 100644
--- a/_includes/code/quickstart.byov.all.ts
+++ b/_includes/code/quickstart.byov.all.ts
@@ -3,11 +3,14 @@
import weaviate, { WeaviateClient } from 'weaviate-client'
import { vectorizer } from 'weaviate-client';
-const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- 'WEAVIATE_INSTANCE_URL', { // Replace WEAVIATE_INSTANCE_URL with your instance URL
- authCredentials: new weaviate.ApiKey('WEAVIATE_INSTANCE_API_KEY'),
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+const openaiKey = process.env.OPENAI_API_KEY as string
+
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
headers: {
- 'X-OpenAI-Api-Key': process.env.OPENAI_API_KEY || '', // Replace with your inference API key
+ 'X-OpenAI-Api-Key': openaiKey, // Replace with your inference API key
}
}
)
diff --git a/_includes/code/quickstart.byov.schema.mdx b/_includes/code/quickstart.byov.schema.mdx
index 5c99819c66..4c9d829338 100644
--- a/_includes/code/quickstart.byov.schema.mdx
+++ b/_includes/code/quickstart.byov.schema.mdx
@@ -39,7 +39,7 @@ import ByovAllShCode from '!!raw-loader!/_includes/code/quickstart.byov.all.sh';
text={ByovAllTsCodeLegacy}
startMarker="// Class definition"
endMarker="// Import data"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/clients.install.mdx b/_includes/code/quickstart/clients.install.mdx
index 19fa10977f..2b911fe007 100644
--- a/_includes/code/quickstart/clients.install.mdx
+++ b/_includes/code/quickstart/clients.install.mdx
@@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
-Install the latest, [Python client `v4`](/developers/weaviate/client-libraries/python), by adding `weaviate-client` to your Python environment with `pip`.
+Install the latest, [Python client `v4`](/developers/weaviate/client-libraries/python), by adding `weaviate-client` to your Python environment with `pip`:
```bash
@@ -14,7 +14,7 @@ pip install -U weaviate-client
-Install the latest, [JS/TS client `v3`](/developers/weaviate/client-libraries/typescript/typescript-v3), by adding `weaviate-client` to your project with `npm`.:
+Install the latest, [JS/TS client `v3`](/developers/weaviate/client-libraries/typescript/typescript-v3), by adding `weaviate-client` to your project with `npm`:
```bash
npm install weaviate-client
diff --git a/_includes/code/quickstart/collection.definition.mdx b/_includes/code/quickstart/collection.definition.mdx
index 768e211c23..ccf63a9eae 100644
--- a/_includes/code/quickstart/collection.definition.mdx
+++ b/_includes/code/quickstart/collection.definition.mdx
@@ -44,7 +44,7 @@ import GoCollectionDefine from '!!raw-loader!/_includes/code/quickstart/go-colle
text={EndToEndTSCodeLegacy}
startMarker="// START CreateCollection"
endMarker="// END CreateCollection"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/connect.withkey.mdx b/_includes/code/quickstart/connect.withkey.mdx
index 444ea5197f..32a17a907e 100644
--- a/_includes/code/quickstart/connect.withkey.mdx
+++ b/_includes/code/quickstart/connect.withkey.mdx
@@ -47,7 +47,7 @@ import GoConnectCode from '!!raw-loader!/_includes/code/quickstart/go-connect.go
text={EndToEndTSCodeLegacy}
startMarker="// InstantiationExample"
endMarker="// END InstantiationExample"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/endtoend.mdx b/_includes/code/quickstart/endtoend.mdx
index fde30475ae..9eb4292300 100644
--- a/_includes/code/quickstart/endtoend.mdx
+++ b/_includes/code/quickstart/endtoend.mdx
@@ -38,7 +38,7 @@ import GoCreateAll from '!!raw-loader!/_includes/code/quickstart/go-create-run-a
text={EndToEndTSCodeLegacy}
startMarker="// EndToEndExample"
endMarker="// END EndToEndExample"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/endtoend.ts b/_includes/code/quickstart/endtoend.ts
index 7eb84e62e1..1bf715c6f6 100644
--- a/_includes/code/quickstart/endtoend.ts
+++ b/_includes/code/quickstart/endtoend.ts
@@ -20,15 +20,16 @@ const client: WeaviateClient = weaviate.client({
// EndToEndExample // InstantiationExample // NearTextExample
import weaviate, { WeaviateClient } from 'weaviate-client';
-const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- process.env.WCD_URL,
- {
- authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY),
- headers: {
- 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY, // Replace with your inference API key
- }
- }
-)
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+const openaiKey = process.env.OPENAI_API_KEY as string
+
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
+ headers: {
+ 'X-OpenAI-Api-Key': openaiKey, // Replace with your inference API key
+ }
+})
// END EndToEndExample // END InstantiationExample // END NearTextExample
diff --git a/_includes/code/quickstart/generativesearch.grouped.mdx b/_includes/code/quickstart/generativesearch.grouped.mdx
index b87850e118..0ee3b96d38 100644
--- a/_includes/code/quickstart/generativesearch.grouped.mdx
+++ b/_includes/code/quickstart/generativesearch.grouped.mdx
@@ -56,7 +56,7 @@ import GoGenGrouped from '!!raw-loader!/_includes/code/quickstart/go-query-gener
text={EndToEndTSCodeLegacy}
startMarker="// GenerativeSearchGroupedTaskExample"
endMarker="// END GenerativeSearchGroupedTaskExample"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/generativesearch.single.mdx b/_includes/code/quickstart/generativesearch.single.mdx
index 6e6a88a895..e32608b002 100644
--- a/_includes/code/quickstart/generativesearch.single.mdx
+++ b/_includes/code/quickstart/generativesearch.single.mdx
@@ -56,7 +56,7 @@ import GoGenerative from '!!raw-loader!/_includes/code/quickstart/go-query-gener
text={EndToEndTSCodeLegacy}
startMarker="// GenerativeSearchExample"
endMarker="// END GenerativeSearchExample"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/import.custom.vectors.mdx b/_includes/code/quickstart/import.custom.vectors.mdx
index eb6d9f29b6..a78d678f46 100644
--- a/_includes/code/quickstart/import.custom.vectors.mdx
+++ b/_includes/code/quickstart/import.custom.vectors.mdx
@@ -32,7 +32,7 @@ import EndToEndTSCode from '!!raw-loader!/_includes/code/quickstart/endtoend.ts'
text={EndToEndTSCode}
startMarker="// Import data with custom vectors"
endMarker="// END Import data with custom vectors"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/import.mdx b/_includes/code/quickstart/import.mdx
index 995477aed1..c11925062a 100644
--- a/_includes/code/quickstart/import.mdx
+++ b/_includes/code/quickstart/import.mdx
@@ -37,7 +37,7 @@ import GoImportObjects from '!!raw-loader!/_includes/code/quickstart/go-add-obje
text={EndToEndTSCodeLegacy}
startMarker="// Import data function"
endMarker="// END Import data function"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/local.quickstart.create_collection.mdx b/_includes/code/quickstart/local.quickstart.create_collection.mdx
new file mode 100644
index 0000000000..7678a3eca8
--- /dev/null
+++ b/_includes/code/quickstart/local.quickstart.create_collection.mdx
@@ -0,0 +1,78 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyCode from '!!raw-loader!/_includes/code/python/local.quickstart.create_collection.py';
+import TSCode from '!!raw-loader!/_includes/code/typescript/local.quickstart.create_collection.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart_local/2_1_create_collection/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/CreateCollection.java';
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+```bash
+curl -X POST \
+-H "Content-Type: application/json" \
+-d '{
+ "class": "Question",
+ "vectorizer": "text2vec-ollama",
+ "moduleConfig": {
+ "text2vec-ollama": {
+ "apiEndpoint": "http://host.docker.internal:11434",
+ "model": "nomic-embed-text",
+ },
+ "generative-ollama": {
+ "apiEndpoint": "http://host.docker.internal:11434",
+ "model": "llama3.2",
+ }
+ }
+}' \
+"http://localhost:8080/v1/schema"
+```
+
+
+
diff --git a/_includes/code/quickstart/local.quickstart.import_objects.mdx b/_includes/code/quickstart/local.quickstart.import_objects.mdx
new file mode 100644
index 0000000000..94a6ef1df7
--- /dev/null
+++ b/_includes/code/quickstart/local.quickstart.import_objects.mdx
@@ -0,0 +1,109 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyCode from '!!raw-loader!/_includes/code/python/local.quickstart.import_objects.py';
+import TSCode from '!!raw-loader!/_includes/code/typescript/local.quickstart.import_objects.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart_local/2_2_add_objects/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/Import.java';
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+:::note
+- Download the `jeopardy_tiny.json` file from [here](https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/jeopardy_tiny.json) before running the following script.
+- This assumes you have `jq` installed.
+:::
+
+```bash
+# Set batch size
+BATCH_ENDPOINT="http://localhost:8080/v1/batch/objects"
+BATCH_SIZE=100
+
+# Read the JSON file and loop through its entries
+lines_processed=0
+batch_data="{\"objects\": ["
+
+cat jeopardy_tiny.json | jq -c '.[]' | while read line; do
+ # Concatenate lines
+ line=$(echo "$line" | jq "{class: \"Question\", properties: {answer: .Answer, question: .Question, category: .Category}}")
+ if [ $lines_processed -eq 0 ]; then
+ batch_data+=$line
+ else
+ batch_data+=",$line"
+ fi
+
+ lines_processed=$((lines_processed + 1))
+
+ # If the batch is full, send it to the API using curl
+ if [ $lines_processed -eq $BATCH_SIZE ]; then
+ batch_data+="]}"
+
+ curl -X POST "$BATCH_ENDPOINT" \
+ -H "Content-Type: application/json" \
+ -d "$batch_data"
+ echo "" # Print a newline for better output formatting
+
+ # Reset the batch data and counter
+ lines_processed=0
+ batch_data="{\"objects\": ["
+ fi
+done
+
+# Send the remaining data (if any) to the API using curl
+if [ $lines_processed -ne 0 ]; then
+ batch_data+="]}"
+
+ curl -X POST "$BATCH_ENDPOINT" \
+ -H "Content-Type: application/json" \
+ -d "$batch_data"
+ echo "" # Print a newline for better output formatting
+fi
+```
+
+
+
diff --git a/_includes/code/quickstart/local.quickstart.is_ready.mdx b/_includes/code/quickstart/local.quickstart.is_ready.mdx
new file mode 100644
index 0000000000..c7959cfdf6
--- /dev/null
+++ b/_includes/code/quickstart/local.quickstart.is_ready.mdx
@@ -0,0 +1,65 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyCode from '!!raw-loader!/_includes/code/python/local.quickstart.is_ready.py';
+import TSCode from '!!raw-loader!/_includes/code/typescript/local.quickstart.is_ready.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart_local/1_is_ready/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/IsReady.java';
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+```bash
+curl -w "\nResponse code: %{http_code}\n" \
+ http://localhost:8080/v1/.well-known/ready
+
+# You should see "Response code: 200" if the instance is ready
+```
+
+
+
diff --git a/_includes/code/quickstart/local.quickstart.query.neartext.mdx b/_includes/code/quickstart/local.quickstart.query.neartext.mdx
new file mode 100644
index 0000000000..0053fef1cf
--- /dev/null
+++ b/_includes/code/quickstart/local.quickstart.query.neartext.mdx
@@ -0,0 +1,81 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyCode from '!!raw-loader!/_includes/code/python/local.quickstart.query.neartext.py';
+import TSCode from '!!raw-loader!/_includes/code/typescript/local.quickstart.query.neartext.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart_local/3_1_neartext/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/NearText.java';
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+```bash
+echo '{
+ "query": "{
+ Get {
+ Question (
+ limit: 2
+ nearText: {
+ concepts: [\"biology\"],
+ }
+ ) {
+ question
+ answer
+ category
+ }
+ }
+ }"
+}' | tr -d "\n" | curl \
+ -X POST \
+ -H 'Content-Type: application/json' \
+ -d @- \
+ http://localhost:8080/v1/graphql
+```
+
+
+
diff --git a/_includes/code/quickstart/local.quickstart.query.rag.mdx b/_includes/code/quickstart/local.quickstart.query.rag.mdx
new file mode 100644
index 0000000000..cdb5fc66c3
--- /dev/null
+++ b/_includes/code/quickstart/local.quickstart.query.rag.mdx
@@ -0,0 +1,93 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyCode from '!!raw-loader!/_includes/code/python/local.quickstart.query.rag.py';
+import TSCode from '!!raw-loader!/_includes/code/typescript/local.quickstart.query.rag.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart_local/3_2_rag/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart_local/RAG.java';
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+```bash
+echo '{
+ "query": "{
+ Get {
+ Question (
+ limit: 2
+ nearText: {
+ concepts: [\"biology\"],
+ }
+ ) {
+ question
+ answer
+ category
+ _additional {
+ generate(
+ groupedResult: {
+ task: \"\"\"
+ Write a tweet with emojis about these facts.
+ \"\"\"
+ }
+ ) {
+ groupedResult
+ error
+ }
+ }
+ }
+ }
+ }"
+}' | tr -d "\n" | curl \
+ -X POST \
+ -H 'Content-Type: application/json' \
+ -d @- \
+ http://localhost:8080/v1/graphql
+```
+
+
+
diff --git a/_includes/code/quickstart/neartext.mdx b/_includes/code/quickstart/neartext.mdx
index a0b4e76f66..9005f10d3f 100644
--- a/_includes/code/quickstart/neartext.mdx
+++ b/_includes/code/quickstart/neartext.mdx
@@ -63,7 +63,7 @@ import GoNearText from '!!raw-loader!/_includes/code/quickstart/go-query-neartex
text={EndToEndTSCodeLegacy}
startMarker="// NearTextExample"
endMarker="// END NearTextExample"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/neartext.where.mdx b/_includes/code/quickstart/neartext.where.mdx
index 03f5e68a09..3e40e37b10 100644
--- a/_includes/code/quickstart/neartext.where.mdx
+++ b/_includes/code/quickstart/neartext.where.mdx
@@ -56,7 +56,7 @@ import GoWhereFilter from '!!raw-loader!/_includes/code/quickstart/go-query-filt
text={EndToEndTSCodeLegacy}
startMarker="// NearTextWhereExample"
endMarker="// END NearTextWhereExample"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/quickstart/quickstart.create_collection.mdx b/_includes/code/quickstart/quickstart.create_collection.mdx
index 465b5d3b52..52a0e0ad32 100644
--- a/_includes/code/quickstart/quickstart.create_collection.mdx
+++ b/_includes/code/quickstart/quickstart.create_collection.mdx
@@ -4,6 +4,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PyCode from '!!raw-loader!/_includes/code/python/quickstart.create_collection.py';
import TSCode from '!!raw-loader!/_includes/code/typescript/quickstart.create_collection.ts';
import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/2_1_create_collection/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/CreateCollection.java';
@@ -40,43 +41,18 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/2_1_cr
title="quickstart/2_1_create_collection/main.go"
/>
-{/*
-```java
-
-package io.weaviate;
-
-import java.util.ArrayList;
-import io.weaviate.client.Config;
-import io.weaviate.client.WeaviateClient;
-import io.weaviate.client.base.Result;
-import io.weaviate.client.v1.schema.model.DataType;
-import io.weaviate.client.v1.schema.model.Property;
-import io.weaviate.client.v1.schema.model.WeaviateClass;
-
-public class App {
- public static void main(String[] args) {
- Config config = new Config("https", "WEAVIATE_INSTANCE_URL/"); // Replace with your Weaviate endpoint
-
- WeaviateClient client = new WeaviateClient(config);
-
- // we will create the class "Question" and the properties
- WeaviateClass clazz = WeaviateClass.builder()
- .className("Question")
- .vectorizer("text2vec-openai")
- .build();
+
+
- // add the schema
- Result result = client.schema().classCreator().withClass(clazz).run();
- if (result.hasErrors()) {
- System.out.println(result.getError());
- return;
- }
- }
-}
-```
+
- */}
```bash
@@ -89,10 +65,10 @@ curl -X POST \
-H "Authorization: Bearer $WCD_API_KEY" \
-d '{
"class": "Question",
- "vectorizer": "text2vec-openai",
+ "vectorizer": "text2vec-cohere",
"moduleConfig": {
- "text2vec-openai": {},
- "generative-openai": {}
+ "text2vec-cohere": {},
+ "generative-cohere": {}
}
}' \
"$WCD_URL/v1/schema"
diff --git a/_includes/code/quickstart/quickstart.import_objects.mdx b/_includes/code/quickstart/quickstart.import_objects.mdx
index b50e684641..330350f4c3 100644
--- a/_includes/code/quickstart/quickstart.import_objects.mdx
+++ b/_includes/code/quickstart/quickstart.import_objects.mdx
@@ -4,6 +4,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PyCode from '!!raw-loader!/_includes/code/python/quickstart.import_objects.py';
import TSCode from '!!raw-loader!/_includes/code/typescript/quickstart.import_objects.ts';
import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/2_2_add_objects/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/Import.java';
@@ -40,43 +41,17 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/2_2_ad
title="quickstart/2_2_import/main.go"
/>
-{/*
-
-```java
-
-package io.weaviate;
-
-import java.util.ArrayList;
-import io.weaviate.client.Config;
-import io.weaviate.client.WeaviateClient;
-import io.weaviate.client.base.Result;
-import io.weaviate.client.v1.schema.model.DataType;
-import io.weaviate.client.v1.schema.model.Property;
-import io.weaviate.client.v1.schema.model.WeaviateClass;
-
-public class App {
- public static void main(String[] args) {
- Config config = new Config("https", "WEAVIATE_INSTANCE_URL/"); // Replace with your Weaviate endpoint
-
- WeaviateClient client = new WeaviateClient(config);
-
- // we will create the class "Question" and the properties
- WeaviateClass clazz = WeaviateClass.builder()
- .className("Question")
- .vectorizer("text2vec-openai")
- .build();
-
- // add the schema
- Result result = client.schema().classCreator().withClass(clazz).run();
- if (result.hasErrors()) {
- System.out.println(result.getError());
- return;
- }
- }
-}
-```
- */}
+
+
+
+
:::note
@@ -88,7 +63,7 @@ public class App {
# Best practice: store your credentials in environment variables
# export WCD_URL="YOUR_INSTANCE_URL" # Your Weaviate instance URL
# export WCD_API_KEY="YOUR_API_KEY" # Your Weaviate instance API key
-# export OPENAI_API_KEY="YOUR_API_KEY" # Your OpenAI API key
+# export COHERE_APIKEY="YOUR_API_KEY" # Your Cohere API key
# Set batch size
BATCH_ENDPOINT="$WCD_URL/v1/batch/objects"
@@ -116,7 +91,7 @@ cat jeopardy_tiny.json | jq -c '.[]' | while read line; do
curl -X POST "$BATCH_ENDPOINT" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $WCD_API_KEY" \
- -H "X-OpenAI-Api-Key: $OPENAI_API_KEY" \
+ -H "X-Cohere-Api-Key: $COHERE_APIKEY" \
-d "$batch_data"
echo "" # Print a newline for better output formatting
@@ -133,7 +108,7 @@ if [ $lines_processed -ne 0 ]; then
curl -X POST "$BATCH_ENDPOINT" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $WCD_API_KEY" \
- -H "X-OpenAI-Api-Key: $OPENAI_API_KEY" \
+ -H "X-Cohere-Api-Key: $COHERE_APIKEY" \
-d "$batch_data"
echo "" # Print a newline for better output formatting
fi
diff --git a/_includes/code/quickstart/quickstart.is_ready.mdx b/_includes/code/quickstart/quickstart.is_ready.mdx
index 9fe6a3a4ed..1532768df0 100644
--- a/_includes/code/quickstart/quickstart.is_ready.mdx
+++ b/_includes/code/quickstart/quickstart.is_ready.mdx
@@ -4,6 +4,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PyCode from '!!raw-loader!/_includes/code/python/quickstart.is_ready.py';
import TSCode from '!!raw-loader!/_includes/code/typescript/quickstart.is_ready.ts';
import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/1_is_ready/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/IsReady.java';
@@ -40,43 +41,17 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/1_is_r
title="quickstart/1_check_readiness/main.go"
/>
-{/*
-```java
-
-package io.weaviate;
-
-import java.util.ArrayList;
-import io.weaviate.client.Config;
-import io.weaviate.client.WeaviateClient;
-import io.weaviate.client.base.Result;
-import io.weaviate.client.v1.schema.model.DataType;
-import io.weaviate.client.v1.schema.model.Property;
-import io.weaviate.client.v1.schema.model.WeaviateClass;
-
-public class App {
- public static void main(String[] args) {
- Config config = new Config("https", "WEAVIATE_INSTANCE_URL/"); // Replace with your Weaviate endpoint
-
- WeaviateClient client = new WeaviateClient(config);
-
- // we will create the class "Question" and the properties
- WeaviateClass clazz = WeaviateClass.builder()
- .className("Question")
- .vectorizer("text2vec-openai")
- .build();
-
- // add the schema
- Result result = client.schema().classCreator().withClass(clazz).run();
- if (result.hasErrors()) {
- System.out.println(result.getError());
- return;
- }
- }
-}
-```
+
+
+
- */}
```bash
diff --git a/_includes/code/quickstart/quickstart.query.neartext.mdx b/_includes/code/quickstart/quickstart.query.neartext.mdx
index e4d76889a2..b429a8855e 100644
--- a/_includes/code/quickstart/quickstart.query.neartext.mdx
+++ b/_includes/code/quickstart/quickstart.query.neartext.mdx
@@ -4,6 +4,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PyCode from '!!raw-loader!/_includes/code/python/quickstart.query.neartext.py';
import TSCode from '!!raw-loader!/_includes/code/typescript/quickstart.query.neartext.ts';
import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/3_1_neartext/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/NearText.java';
@@ -40,50 +41,24 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/3_1_ne
title="quickstart/3_1_neartext/main.go"
/>
-{/*
-```java
-
-package io.weaviate;
-
-import java.util.ArrayList;
-import io.weaviate.client.Config;
-import io.weaviate.client.WeaviateClient;
-import io.weaviate.client.base.Result;
-import io.weaviate.client.v1.schema.model.DataType;
-import io.weaviate.client.v1.schema.model.Property;
-import io.weaviate.client.v1.schema.model.WeaviateClass;
-
-public class App {
- public static void main(String[] args) {
- Config config = new Config("https", "WEAVIATE_INSTANCE_URL/"); // Replace with your Weaviate endpoint
-
- WeaviateClient client = new WeaviateClient(config);
-
- // we will create the class "Question" and the properties
- WeaviateClass clazz = WeaviateClass.builder()
- .className("Question")
- .vectorizer("text2vec-openai")
- .build();
-
- // add the schema
- Result result = client.schema().classCreator().withClass(clazz).run();
- if (result.hasErrors()) {
- System.out.println(result.getError());
- return;
- }
- }
-}
-```
+
+
+
- */}
```bash
# Best practice: store your credentials in environment variables
# export WCD_URL="YOUR_INSTANCE_URL" # Your Weaviate instance URL
# export WCD_API_KEY="YOUR_API_KEY" # Your Weaviate instance API key
-# export OPENAI_API_KEY="YOUR_API_KEY" # Your OpenAI API key
+# export COHERE_APIKEY="YOUR_API_KEY" # Your Cohere API key
echo '{
"query": "{
@@ -104,7 +79,7 @@ echo '{
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $WCD_API_KEY" \
- -H "X-OpenAI-Api-Key: $OPENAI_API_KEY" \
+ -H "X-Cohere-Api-Key: $COHERE_APIKEY" \
-d @- \
$WCD_URL/v1/graphql
```
diff --git a/_includes/code/quickstart/quickstart.query.rag.mdx b/_includes/code/quickstart/quickstart.query.rag.mdx
index 54ebf847f2..d97a28c7a4 100644
--- a/_includes/code/quickstart/quickstart.query.rag.mdx
+++ b/_includes/code/quickstart/quickstart.query.rag.mdx
@@ -4,6 +4,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PyCode from '!!raw-loader!/_includes/code/python/quickstart.query.rag.py';
import TSCode from '!!raw-loader!/_includes/code/typescript/quickstart.query.rag.ts';
import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/3_2_rag/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/quickstart/RAG.java';
@@ -40,50 +41,24 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/quickstart/3_2_ra
title="quickstart/3_2_rag/main.go"
/>
-{/*
-```java
-
-package io.weaviate;
-
-import java.util.ArrayList;
-import io.weaviate.client.Config;
-import io.weaviate.client.WeaviateClient;
-import io.weaviate.client.base.Result;
-import io.weaviate.client.v1.schema.model.DataType;
-import io.weaviate.client.v1.schema.model.Property;
-import io.weaviate.client.v1.schema.model.WeaviateClass;
-
-public class App {
- public static void main(String[] args) {
- Config config = new Config("https", "WEAVIATE_INSTANCE_URL/"); // Replace with your Weaviate endpoint
-
- WeaviateClient client = new WeaviateClient(config);
-
- // we will create the class "Question" and the properties
- WeaviateClass clazz = WeaviateClass.builder()
- .className("Question")
- .vectorizer("text2vec-openai")
- .build();
-
- // add the schema
- Result result = client.schema().classCreator().withClass(clazz).run();
- if (result.hasErrors()) {
- System.out.println(result.getError());
- return;
- }
- }
-}
-```
+
+
+
- */}
```bash
# Best practice: store your credentials in environment variables
# export WCD_URL="YOUR_INSTANCE_URL" # Your Weaviate instance URL
# export WCD_API_KEY="YOUR_API_KEY" # Your Weaviate instance API key
-# export OPENAI_API_KEY="YOUR_API_KEY" # Your OpenAI API key
+# export COHERE_APIKEY="YOUR_API_KEY" # Your Cohere API key
echo '{
"query": "{
@@ -116,7 +91,7 @@ echo '{
-X POST \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $WCD_API_KEY" \
- -H "X-OpenAI-Api-Key: $OPENAI_API_KEY" \
+ -H "X-Cohere-Api-Key: $COHERE_APIKEY" \
-d @- \
$WCD_URL/v1/graphql
```
diff --git a/_includes/code/schema.things.create.replication.mdx b/_includes/code/schema.things.create.replication.mdx
index d43d186099..e665958f52 100644
--- a/_includes/code/schema.things.create.replication.mdx
+++ b/_includes/code/schema.things.create.replication.mdx
@@ -22,7 +22,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
text={PyCodeV3}
startMarker="# START ReplicationSettings"
endMarker="# END ReplicationSettings"
- language="py"
+ language="pyv3"
/>
@@ -40,7 +40,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
text={TSCodeLegacy}
startMarker="// START ReplicationSettings"
endMarker="// END ReplicationSettings"
- language="ts"
+ language="tsv2"
/>
diff --git a/_includes/code/starter-guides/custom-vectors-ts3.ts b/_includes/code/starter-guides/custom-vectors-ts3.ts
index 64e3dc8d01..b09ab3978b 100644
--- a/_includes/code/starter-guides/custom-vectors-ts3.ts
+++ b/_includes/code/starter-guides/custom-vectors-ts3.ts
@@ -16,11 +16,11 @@ const WCD_API_KEY=process.env["WCD_API_KEY"];
const OPENAI_API_KEY=process.env["OPENAI_API_KEY"];
const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
- WCD_URL,
+ WCD_URL as string,
{
- authCredentials: new weaviate.ApiKey(WCD_API_KEY),
+ authCredentials: new weaviate.ApiKey(WCD_API_KEY as string),
headers: {
- 'X-OpenAI-Api-Key': OPENAI_API_KEY,
+ 'X-OpenAI-Api-Key': OPENAI_API_KEY as string,
}
}
)
diff --git a/_includes/code/typescript/local.quickstart.create_collection.ts b/_includes/code/typescript/local.quickstart.create_collection.ts
new file mode 100644
index 0000000000..2ca5b25d8f
--- /dev/null
+++ b/_includes/code/typescript/local.quickstart.create_collection.ts
@@ -0,0 +1,26 @@
+// CreateCollection
+import weaviate, { WeaviateClient, vectorizer, generative } from 'weaviate-client';
+
+const client: WeaviateClient = await weaviate.connectToLocal();
+
+// END CreateCollection
+// Delete this collection if it already exists
+await client.collections.delete('Question');
+
+// CreateCollection
+// highlight-start
+await client.collections.create({
+ name: 'Question',
+ vectorizers: vectorizer.text2VecOllama({ // Configure the Ollama embedding integration
+ apiEndpoint: 'http://host.docker.internal:11434', // Allow Weaviate from within a Docker container to contact your Ollama instance
+ model: 'nomic-embed-text', // The model to use
+ }),
+ generative: generative.ollama({ // Configure the Ollama generative integration
+ apiEndpoint: 'http://host.docker.internal:11434', // Allow Weaviate from within a Docker container to contact your Ollama instance
+ model: 'llama3.2', // The model to use
+ }),
+});
+// highlight-end
+
+client.close(); // Close the client connection
+// END CreateCollection
diff --git a/_includes/code/typescript/local.quickstart.import_objects.ts b/_includes/code/typescript/local.quickstart.import_objects.ts
new file mode 100644
index 0000000000..910c239832
--- /dev/null
+++ b/_includes/code/typescript/local.quickstart.import_objects.ts
@@ -0,0 +1,28 @@
+// Import
+import weaviate, { WeaviateClient } from 'weaviate-client';
+
+const client: WeaviateClient = await weaviate.connectToLocal();
+
+// Load data
+async function getJsonData() {
+ const file = await fetch(
+ 'https://raw.githubusercontent.com/weaviate-tutorials/quickstart/main/data/jeopardy_tiny.json'
+ );
+ return file.json();
+}
+
+// highlight-start
+// Note: The TS client does not have a `batch` method yet
+// We use `insertMany` instead, which sends all of the data in one request
+async function importQuestions() {
+ const questions = client.collections.get('Question');
+ const data = await getJsonData();
+ const result = await questions.data.insertMany(data);
+ console.log('Insertion response: ', result);
+}
+
+await importQuestions();
+// highlight-end
+
+client.close(); // Close the client connection
+// END Import
diff --git a/_includes/code/typescript/local.quickstart.is_ready.ts b/_includes/code/typescript/local.quickstart.is_ready.ts
new file mode 100644
index 0000000000..f218b7b6ee
--- /dev/null
+++ b/_includes/code/typescript/local.quickstart.is_ready.ts
@@ -0,0 +1,13 @@
+// InstantiationExample
+import weaviate, { WeaviateClient } from 'weaviate-client';
+
+// highlight-start
+const client: WeaviateClient = await weaviate.connectToLocal();
+// highlight-end
+
+var clientReadiness = await client.isReady();
+
+console.log(clientReadiness); // Should return `true`
+
+client.close(); // Close the client connection
+// END InstantiationExample
diff --git a/_includes/code/typescript/local.quickstart.query.neartext.ts b/_includes/code/typescript/local.quickstart.query.neartext.ts
new file mode 100644
index 0000000000..8692e96cd5
--- /dev/null
+++ b/_includes/code/typescript/local.quickstart.query.neartext.ts
@@ -0,0 +1,19 @@
+// NearText
+import weaviate, { WeaviateClient } from 'weaviate-client';
+
+const client: WeaviateClient = await weaviate.connectToLocal();
+
+// highlight-start
+const questions = client.collections.get('Question');
+
+const result = await questions.query.nearText('biology', {
+ limit: 2,
+});
+// highlight-end
+
+result.objects.forEach((item) => {
+ console.log(JSON.stringify(item.properties, null, 2));
+});
+
+client.close(); // Close the client connection
+// END NearText
diff --git a/_includes/code/typescript/local.quickstart.query.rag.ts b/_includes/code/typescript/local.quickstart.query.rag.ts
new file mode 100644
index 0000000000..98e6287f84
--- /dev/null
+++ b/_includes/code/typescript/local.quickstart.query.rag.ts
@@ -0,0 +1,23 @@
+// RAG
+import weaviate, { WeaviateClient } from 'weaviate-client';
+
+const client: WeaviateClient = await weaviate.connectToLocal();
+
+// highlight-start
+const questions = client.collections.get('Question');
+
+const result = await questions.generate.nearText(
+ 'biology',
+ {
+ groupedTask: 'Write a tweet with emojis about these facts.',
+ },
+ {
+ limit: 2,
+ }
+);
+// highlight-end
+
+console.log(result.generated);
+
+client.close(); // Close the client connection
+// END RAG
diff --git a/_includes/code/typescript/quickstart.create_collection.ts b/_includes/code/typescript/quickstart.create_collection.ts
index 5d5b2b4498..695f58c3f1 100644
--- a/_includes/code/typescript/quickstart.create_collection.ts
+++ b/_includes/code/typescript/quickstart.create_collection.ts
@@ -20,8 +20,8 @@ await client.collections.delete('Question');
// highlight-start
await client.collections.create({
name: 'Question',
- vectorizers: vectorizer.text2VecOpenAI(),
- generative: generative.openAI(),
+ vectorizers: vectorizer.text2VecCohere(),
+ generative: generative.cohere(),
});
// highlight-end
diff --git a/_includes/code/typescript/quickstart.import_objects.ts b/_includes/code/typescript/quickstart.import_objects.ts
index 4636c907d5..da6fbbbc97 100644
--- a/_includes/code/typescript/quickstart.import_objects.ts
+++ b/_includes/code/typescript/quickstart.import_objects.ts
@@ -4,7 +4,7 @@ import weaviate, { WeaviateClient } from 'weaviate-client';
// Best practice: store your credentials in environment variables
const wcdUrl = process.env.WCD_URL as string;
const wcdApiKey = process.env.WCD_API_KEY as string;
-const openAIKey = process.env.OPENAI_APIKEY as string;
+const cohereKey = process.env.COHERE_APIKEY as string;
const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
wcdUrl, // Replace with your Weaviate Cloud URL
@@ -12,7 +12,7 @@ const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
authCredentials: new weaviate.ApiKey(wcdApiKey), // Replace with your Weaviate Cloud API key
// highlight-start
headers: {
- 'X-OpenAI-Api-Key': openAIKey, // Replace with your OpenAI API key
+ 'X-Cohere-Api-Key': cohereKey, // Replace with your Cohere API key
},
// highlight-end
}
diff --git a/_includes/code/typescript/quickstart.is_ready.ts b/_includes/code/typescript/quickstart.is_ready.ts
index 4ce963b68b..98ed119363 100644
--- a/_includes/code/typescript/quickstart.is_ready.ts
+++ b/_includes/code/typescript/quickstart.is_ready.ts
@@ -5,12 +5,14 @@ import weaviate, { WeaviateClient } from 'weaviate-client';
const wcdUrl = process.env.WCD_URL as string;
const wcdApiKey = process.env.WCD_API_KEY as string;
+// highlight-start
const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
wcdUrl, // Replace with your Weaviate Cloud URL
{
authCredentials: new weaviate.ApiKey(wcdApiKey), // Replace with your Weaviate Cloud API key
}
);
+// highlight-end
var clientReadiness = await client.isReady();
console.log(clientReadiness); // Should return `true`
diff --git a/_includes/code/typescript/quickstart.query.neartext.ts b/_includes/code/typescript/quickstart.query.neartext.ts
index e8d1c66351..621e6064bf 100644
--- a/_includes/code/typescript/quickstart.query.neartext.ts
+++ b/_includes/code/typescript/quickstart.query.neartext.ts
@@ -4,7 +4,7 @@ import weaviate, { WeaviateClient } from 'weaviate-client';
// Best practice: store your credentials in environment variables
const wcdUrl = process.env.WCD_URL as string;
const wcdApiKey = process.env.WCD_API_KEY as string;
-const openAIKey = process.env.OPENAI_APIKEY as string;
+const cohereKey = process.env.COHERE_APIKEY as string;
const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
wcdUrl, // Replace with your Weaviate Cloud URL
@@ -12,7 +12,7 @@ const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
authCredentials: new weaviate.ApiKey(wcdApiKey), // Replace with your Weaviate Cloud API key
// highlight-start
headers: {
- 'X-OpenAI-Api-Key': openAIKey, // Replace with your OpenAI API key
+ 'X-Cohere-Api-Key': cohereKey, // Replace with your Cohere API key
},
// highlight-end
}
diff --git a/_includes/code/typescript/quickstart.query.rag.ts b/_includes/code/typescript/quickstart.query.rag.ts
index 3a32fb9dc5..ed2d16aa08 100644
--- a/_includes/code/typescript/quickstart.query.rag.ts
+++ b/_includes/code/typescript/quickstart.query.rag.ts
@@ -4,7 +4,7 @@ import weaviate, { WeaviateClient } from 'weaviate-client';
// Best practice: store your credentials in environment variables
const wcdUrl = process.env.WCD_URL as string;
const wcdApiKey = process.env.WCD_API_KEY as string;
-const openAIKey = process.env.OPENAI_APIKEY as string;
+const cohereKey = process.env.COHERE_APIKEY as string;
const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
wcdUrl, // Replace with your Weaviate Cloud URL
@@ -12,7 +12,7 @@ const client: WeaviateClient = await weaviate.connectToWeaviateCloud(
authCredentials: new weaviate.ApiKey(wcdApiKey), // Replace with your Weaviate Cloud API key
// highlight-start
headers: {
- 'X-OpenAI-Api-Key': openAIKey, // Replace with your OpenAI API key
+ 'X-Cohere-Api-Key': cohereKey, // Replace with your Cohere API key
},
// highlight-end
}
diff --git a/_includes/collection-mutable-parameters.mdx b/_includes/collection-mutable-parameters.mdx
index 42570cbd77..0dbde5d731 100644
--- a/_includes/collection-mutable-parameters.mdx
+++ b/_includes/collection-mutable-parameters.mdx
@@ -8,6 +8,10 @@
- `additions`
- `preset`
- `removals`
+- `moduleConfig` (generative & reranker modules only, from `1.26.8` and `v1.27.1`)
+- `multiTenancyConfig`
+ - `autoTenantCreation` (introduced in `v1.25.0`)
+ - `autoTenantActivation` (introduced in `v1.25.2`)
- `replicationConfig`
- `asyncEnabled` (introduced in `v1.26.0`)
- `factor` (not mutable in `v1.25` or higher)
@@ -28,6 +32,3 @@
- `distribution`
- `skip`
- `vectorCacheMaxObjects`
-- `MultiTenancyConfig`
- - `autoTenantCreation` (introduced in `v1.25.0`)
- - `autoTenantActivation` (introduced in `v1.25.2`)
diff --git a/_includes/gcp.token.expiry.notes.mdx b/_includes/gcp.token.expiry.notes.mdx
index 0b33a80090..74a8a0cd08 100644
--- a/_includes/gcp.token.expiry.notes.mdx
+++ b/_includes/gcp.token.expiry.notes.mdx
@@ -39,7 +39,7 @@ def re_instantiate_weaviate() -> weaviate.Client:
client = weaviate.Client(
url = "https://WEAVIATE_INSTANCE_URL", # Replace WEAVIATE_INSTANCE_URL with the URL
additional_headers = {
- "X-Google-Vertex-Api-Key": token,
+ "X-Goog-Vertex-Api-Key": token,
}
)
return client
@@ -98,7 +98,7 @@ def re_instantiate_weaviate() -> weaviate.Client:
cluster_url="https://WEAVIATE_INSTANCE_URL", # Replace WEAVIATE_INSTANCE_URL with the URL
auth_credentials=Auth.api_key(weaviate_api_key), # Replace with your Weaviate Cloud key
headers={
- "X-Google-Vertex-Api-Key": token,
+ "X-Goog-Vertex-Api-Key": token,
},
)
return client
diff --git a/_includes/mutable-generative-config.md b/_includes/mutable-generative-config.md
new file mode 100644
index 0000000000..62b56d904f
--- /dev/null
+++ b/_includes/mutable-generative-config.md
@@ -0,0 +1,3 @@
+:::info Generative model integration mutability
+A collection's `generative` model integration configuration is mutable from `v1.25.23`, `v1.26.8` and `v1.27.1`. See [this section](/developers/weaviate/manage-data/collections#update-the-generative-model-integration) for details on how to update the collection configuration.
+:::
diff --git a/_includes/mutable-reranker-config.md b/_includes/mutable-reranker-config.md
new file mode 100644
index 0000000000..bbbe9eaf1d
--- /dev/null
+++ b/_includes/mutable-reranker-config.md
@@ -0,0 +1,3 @@
+:::info Generative model integration mutable from `v1.25.23`, `v1.26.8` and `v1.27.1`
+A collection's `reranker` model integration configuration is mutable from `v1.25.23`, `v1.26.8` and `v1.27.1`. See [this section](/developers/weaviate/manage-data/collections#update-the-reranker-model-integration) for details on how to update the collection configuration.
+:::
diff --git a/_includes/release-history.md b/_includes/release-history.md
new file mode 100644
index 0000000000..ae1929ad77
--- /dev/null
+++ b/_includes/release-history.md
@@ -0,0 +1,45 @@
+This table lists the Weaviate core versions and corresponding client library versions.
+
+| Weaviate ([GitHub][cWeaviate]) | First release date | Python ([GitHub][cPython]) | TypeScript/ JavaScript ([GitHub][cTypeScript]) | Go ([GitHub][cGo]) | Java ([GitHub][cJava]) |
+| :- | :- | :-: | :-: | :-: | :-: |
+| 1.28.x | 2024-12-11 | 4.10.x | TBC | TBC | TBC |
+| 1.27.x | 2024-10-16 | 4.9.x | 3.2.x | 4.16.x | 5.0.x 4.9.x |
+| 1.26.x | 2024-07-22 | 4.7.x | 3.1.x | 4.15.x | 4.8.x |
+| 1.25.x | 2024-05-10 | 4.6.x | 2.1.x | 4.13.x | 4.6.x |
+| 1.24.x | 2024-02-27 | 4.5.x | 2.0.x | 4.10.x | 4.4.x |
+| 1.23.x | 2023-12-18 | 3.26.x | 1.5.x | 4.10.x | 4.4.x |
+| 1.22.x | 2023-10-27 | 3.25.x | 1.5.x | 4.10.x | 4.3.x |
+| 1.21.x | 2023-08-17 | 3.22.x | 1.4.x | 4.9.x | 4.2.x |
+| 1.20.x | 2023-07-06 | 3.22.x | 1.1.x | 4.7.x | 4.2.x |
+| 1.19.x | 2023-05-04 | 3.17.x | 1.1.x[1](#typescript-client-change)| 4.7.x | 4.0.x |
+| 1.18.x | 2023-03-07 | 3.13.x | 2.14.x | 4.6.x | 3.6.x |
+| 1.17.x | 2022-12-20 | 3.9.x | 2.14.x | 4.5.x | 3.5.x |
+| 1.16.x | 2022-10-31 | 3.8.x | 2.13.x | 4.4.x | 3.4.x |
+| 1.15.x | 2022-09-07 | 3.6.x | 2.12.x | 4.3.x | 3.3.x |
+| 1.14.x | 2022-07-07 | 3.6.x | 2.11.x | 4.2.x | 3.2.x |
+| 1.13.x | 2022-05-03 | 3.4.x | 2.9.x | 4.0.x | 2.4.x |
+| 1.12.x | 2022-04-05 | 3.4.x | 2.8.x | 3.0.x | 2.3.x |
+| 1.11.x | 2022-03-14 | 3.2.x | 2.7.x | 2.6.x | 2.3.x |
+| 1.10.x | 2022-01-27 | 3.1.x | 2.5.x | 2.4.x | 2.1.x |
+| 1.9.x | 2021-12-10 | 3.1.x | 2.4.x | 2.4.x | 2.1.x |
+| 1.8.x | 2021-11-30 | 3.1.x | 2.4.x | 2.3.x | 1.1.x |
+| 1.7.x | 2021-09-01 | 3.1.x | 2.4.x | 2.3.x | 1.1.x |
+| 1.6.x | 2021-08-11 | 2.4.x | 2.3.x | 2.2.x | 1.0.x |
+| 1.5.x | 2021-07-13 | 2.2.x | 2.1.x | 2.1.x | 1.0.x |
+| 1.4.x | 2021-06-09 | 2.2.x | 2.1.x | 2.1.x | 1.0.x |
+| 1.3.x | 2021-04-23 | 2.2.x | 2.1.x | 2.1.x | 1.0.x |
+| 1.2.x | 2021-03-15 | 2.2.x | 2.0.x | 1.1.x | - |
+| 1.1.x | 2021-02-10 | 2.1.x | - | - | - |
+| 1.0.x | 2021-01-14 | 2.0.x | - | - | - |
+
+#### TypeScript client change
+
+The [TypeScript client](https://github.com/weaviate/typescript-client) replaced the [JavaScript client](https://github.com/weaviate/weaviate-javascript-client) on 2023-03-17.
+
+[comment]: # ( repo links )
+
+[cWeaviate]: https://github.com/weaviate/weaviate/releases
+[cPython]: https://github.com/weaviate/weaviate-python-client/releases
+[cTypeScript]: https://github.com/weaviate/typescript-client/releases
+[cGo]: https://github.com/weaviate/weaviate-go-client/releases
+[cJava]: https://github.com/weaviate/java-client/releases
diff --git a/_includes/tokenization_definition.mdx b/_includes/tokenization_definition.mdx
index 616a9e4d2c..5127a6b0bd 100644
--- a/_includes/tokenization_definition.mdx
+++ b/_includes/tokenization_definition.mdx
@@ -6,4 +6,5 @@
| `field` | Index the whole field after trimming whitespace characters. | `Hello, (beautiful) world` |
| `trigram` | Split the property as rolling trigrams. | `Hel`, `ell`, `llo`, `lo,`, ... |
| `gse` | Use the `gse` tokenizer to split the property. | [See `gse` docs](https://pkg.go.dev/github.com/go-ego/gse#section-readme) |
-| `kagome_kr` | Use the `Kagome` tokenizer with a Korean dictionary to split the property. | [See `kagome` docs](https://github.com/ikawaha/kagome) and the [Korean dictionary](https://github.com/ikawaha/kagome-dict-ko) |
+| `kagome_ja` | Use the `Kagome` tokenizer with a Japanese (IPA) dictionary to split the property. | [See `kagome` docs](https://github.com/ikawaha/kagome) and the [dictionary](https://github.com/ikawaha/kagome-dict/). |
+| `kagome_kr` | Use the `Kagome` tokenizer with a Korean dictionary to split the property. | [See `kagome` docs](https://github.com/ikawaha/kagome) and the [Korean dictionary](https://github.com/ikawaha/kagome-dict-ko). |
diff --git a/apple-and-weaviate/2024-07-02-create-and-load-a-weaviate-cluster/index.mdx b/apple-and-weaviate/2024-07-02-create-and-load-a-weaviate-cluster/index.mdx
index 3fcee03133..332596aca4 100644
--- a/apple-and-weaviate/2024-07-02-create-and-load-a-weaviate-cluster/index.mdx
+++ b/apple-and-weaviate/2024-07-02-create-and-load-a-weaviate-cluster/index.mdx
@@ -9,12 +9,9 @@ image: ./img/hero.png
---
-
-## Overview
-
In this article we will setup a Weaviate Cluster and load the same books data used in the article [IOS Intro - Search with Benefits](/apple-and-weaviate/apple-apps-part-1).
-You can download the resources used in this blog [here](https://github.com/weaviate/BookRecs/blob/main/blog-resources/article05.zip).
+You can download the resources used in this blog [here](https://github.com/weaviate/BookRecs/blob/main/blog-resources/article05.zip).
### Quckstart
@@ -50,11 +47,11 @@ Create an Xcode Project using External Builder as described in **External Builde
-## Install Weaviate Client
+## Install Weaviate Client
From the root directory of your Xcode Project, install the Weaviate Client.
- pip install -U weaviate-client
+ pip install -U weaviate-client
Refer to the section **Install Python Packages and Setup a Virtual Environment** in the article **External Builder (Xcode) for Python** for guidance
diff --git a/apple-and-weaviate/2024-07-02-external-builder-xcode-for-python/index.mdx b/apple-and-weaviate/2024-07-02-external-builder-xcode-for-python/index.mdx
index c031219e6c..84ee2d2a7f 100644
--- a/apple-and-weaviate/2024-07-02-external-builder-xcode-for-python/index.mdx
+++ b/apple-and-weaviate/2024-07-02-external-builder-xcode-for-python/index.mdx
@@ -9,8 +9,6 @@ image: ./img/external-builder-hero.png
---
-## Overview
-
Xcode itself isn't built for directly developing and building Python projects. However, you can leverage external builders to integrate Python development into your Xcode workflow.
Using Xcode to run python scripts is an alternative to using a native to python playground like Jupyter while allowing you to use a development tool you are familiar with.
diff --git a/apple-and-weaviate/2024-07-02-running-python-on-your-mac/index.mdx b/apple-and-weaviate/2024-07-02-running-python-on-your-mac/index.mdx
index c68203cb87..2777bd70eb 100644
--- a/apple-and-weaviate/2024-07-02-running-python-on-your-mac/index.mdx
+++ b/apple-and-weaviate/2024-07-02-running-python-on-your-mac/index.mdx
@@ -9,8 +9,6 @@ image: ./img/python-on-mac-hero.png
---
-## Overview
-
Weaviate and other data science frameworks are making Python a powerful tool for data analysis on the Mac. Get ready to use them in your workflow.
@@ -32,7 +30,7 @@ There are several ways to get Python up and running on your Mac:
1. Official Python Installer: Download the latest version directly from the [Python website](https://www.python.org/downloads/). This is a straightforward approach.
-2. [Anaconda](https://anaconda.cloud/getting-started): This popular distribution includes Python along with a vast collection of scientific computing and data science packages. This method simplifies installation and is recommended.
+2. [Anaconda](https://anaconda.cloud/getting-started): This popular distribution includes Python along with a vast collection of scientific computing and data science packages. This method simplifies installation and is recommended.
3. Homebrew: This package manager allows you to install various software on your Mac, including Python. It offers flexibility for managing multiple Python versions.
@@ -54,8 +52,8 @@ Consider having a project01 directory that includes a volume calculator applicat
Let's walk through setting up a virtual environment for your Python project, assuming you already have user installed version of Python in place. We'll use the project01 directory as an example:
-Navigate to the project directory: Open your terminal and use the cd command to switch to the directory containing your project files. For instance, if project01 is on your desktop, you might type...
-
+Navigate to the project directory: Open your terminal and use the cd command to switch to the directory containing your project files. For instance, if project01 is on your desktop, you might type...
+
cd ~/Desktop/project01.
**Create the virtual environment**: Use the following command in your terminal, replacing the last venv parameter with your desired virtual environment name:
@@ -65,7 +63,7 @@ Navigate to the project directory: Open your terminal and use the cd command to
This creates a new directory named venv in your project directory. It contains all the necessary files for a separate Python environment.
-**Activate the virtual environment**: Tell your terminal to use the newly created virtual environment.
+**Activate the virtual environment**: Tell your terminal to use the newly created virtual environment.
source venv/bin/activate
@@ -91,15 +89,15 @@ These commands help you verify your Python setup:
**Version of Python**:
-This will display the version of Python currently in use by your terminal.
+This will display the version of Python currently in use by your terminal.
On my Mac, it currently shows Python 3.11.7.
- python --version
+ python --version
**Path to Python Executable**:
-This will show the location of the Python executable (via an alias) that your terminal is using.
+This will show the location of the Python executable (via an alias) that your terminal is using.
which python
diff --git a/authors.yml b/authors.yml
index 9454412f00..213d9c3f2e 100644
--- a/authors.yml
+++ b/authors.yml
@@ -7,11 +7,15 @@ alea:
name: Alea Abed
title: Head of Marketing
image_url: /img/people/icon/alea.jpg
+alvin:
+ name: Alvin Richards
+ title: Head of Product
+ image_url: /img/people/icon/alvin.jpg
ara:
name: Ara Gonzalez
title: People Operations Manager
url: https://linkedin.com/in/aranzazugonzalez
- image_url: /img/people/icon/ara.png
+ image_url: /img/people/icon/ara.jpg
bob:
name: Bob van Luijt
title: CEO
@@ -21,7 +25,7 @@ byron:
name: Byron Voorbach
title: Head of Customer Success
url: https://github.com/byronvoorbach
- image_url: /img/people/icon/byron.png
+ image_url: /img/people/icon/byron.jpg
connor:
name: Connor Shorten
title: Product Researcher
@@ -61,12 +65,12 @@ ieva:
name: Ieva Urbaite
title: Marketing
url: https://www.linkedin.com/in/ievaurbaite/
- image_url: /img/people/icon/ieva.png
+ image_url: /img/people/icon/ieva.jpg
jessie:
name: Jessie de Groot
title: Head of People and Culture
url: https://linkedin.com/in/jessiedegroot/
- image_url: /img/people/icon/jessie.png
+ image_url: /img/people/icon/jessie.jpg
jp:
name: Joon-Pil (JP) Hwang
title: Educator
@@ -79,7 +83,7 @@ laura:
image_url: /img/people/icon/laura.jpg
leonie:
name: Leonie Monigatti
- title: Developer Advocate
+ title: Machine Learning Engineer
url: https://www.linkedin.com/in/804250ab/
image_url: /img/people/icon/leonie.jpg
marion:
@@ -91,7 +95,7 @@ mary:
name: Mary Newhauser
title: Machine Learning Engineer
url: https://www.linkedin.com/in/mary-newhauser-02273551/
- image_url: /img/people/icon/mary.png
+ image_url: /img/people/icon/mary.jpg
pete:
name: Peter Schramm
title: WCD Tech Lead
@@ -111,7 +115,7 @@ svitlana:
name: Svitlana Smolianova
title: Head of Design
url: https://linkedin.com/in/svitlana-sm
- image_url: /img/people/icon/svitlana.png
+ image_url: /img/people/icon/svitlana.jpg
zain:
name: Zain Hasan
title: Developer Advocate
@@ -121,12 +125,12 @@ john:
name: John Trengrove
title: Head of Applied Research
url: https://github.com/trengrj
- image_url: /img/people/icon/john.png
+ image_url: /img/people/icon/john.jpg
shukri:
name: Mohd Shukri Hasan
title: Integration Solution Engineer
url: https://github.com/hsm207
- image_url: /img/people/icon/shukri.jpeg
+ image_url: /img/people/icon/shukri.jpg
tommy:
name: Tommy Smith
title: Polyglot Engineer
@@ -136,12 +140,12 @@ jerry:
name: Jerry Liu
title: Co-Founder/CEO LlamaIndex
url: https://github.com/jerryjliu
- image_url: /img/people/icon/jerry.png
+ image_url: /img/people/icon/jerry.jpg
sam:
name: Sam Stoelinga
title: Contributor to Substratus.AI
url: https://github.com/samos123
- image_url: /img/people/icon/sam.png
+ image_url: /img/people/icon/sam.jpg
femke:
name: Femke Plantinga
title: Content Manager
@@ -159,12 +163,12 @@ victoria:
name: Victoria Slocum
title: Machine Learning Engineer
url: https://www.linkedin.com/in/victorialslocum/
- image_url: /img/people/icon/victoria.png
+ image_url: /img/people/icon/victoria.jpg
adam:
name: Adam Chan
title: Developer Growth Engineer
url: https://www.linkedin.com/in/itsajchan/
- image_url: /img/people/icon/adam.png
+ image_url: /img/people/icon/adam.jpg
asdine:
name: Asdine El Hrychy
title: Software Engineer
@@ -174,22 +178,32 @@ ajit:
name: Ajit Mistry
title: Developer Growth Intern
url: https://github.com/ajit283
- image_url: /img/people/icon/ajit.png
+ image_url: /img/people/icon/ajit.jpg
amir:
name: Amir Houieh
title: Founder & CEO of Unbody
url: https://www.unbody.io/
- image_url: /img/people/icon/amir.jpeg
+ image_url: /img/people/icon/amir.jpg
+hossein:
+ name: Hossein Mehrabi
+ title: Founder & CTO of Unbody
+ url: https://github.com/jeangovil
+ image_url: /img/people/icon/hossein.jpg
charles:
name: Charles Frye
title: AI Engineer
url: https://twitter.com/charles_irl
- image_url: /img/people/icon/charles-modal.png
+ image_url: /img/people/icon/charles-modal.jpg
randy:
name: Randy Fong
title: Senior iOS Developer
url: https://www.linkedin.com/in/randy-fong-1530545/
- image_url: /img/people/icon/randy.jpeg
+ image_url: /img/people/icon/randy.jpg
+aman:
+ name: Aman Sharma
+ title: CTO @Lamatic.ai
+ url: https://www.linkedin.com/in/amanintech
+ image_url: /img/people/icon/aman.jpg
charles-pierse:
name: Charles Pierse
title: Head of Weaviate Labs
@@ -204,5 +218,29 @@ deepti:
name: Deepti Naidu
title: Partner Solution Engineer
url: https://www.linkedin.com/in/deeptinaidu/
- image_url: /img/people/icon/deepti.png
-
+ image_url: /img/people/icon/deepti.jpg
+tony:
+ name: Tony Le
+ title: Business Development Director
+ url: https://www.linkedin.com/in/tony-l-35a305b/
+ image_url: /img/people/icon/tony.jpg
+jobi:
+ name: Jobi George
+ title: Head of Partnerships
+ url: https://www.linkedin.com/in/jobigeorge/
+ image_url: /img/people/icon/jobi.jpg
+spiros:
+ name: Spiros
+ title: Head of Security
+ url: https://github.com/spiros-spiros
+ image_url: /img/people/icon/spiros.jpg
+staci:
+ name: Staci Blue
+ title: Event Marketing Manager
+ url: https://www.linkedin.com/in/staci-blue/
+ image_url: /img/people/icon/staci.jpg
+charly:
+ name: Charly Poly
+ title: Developer Marketer at Inngest
+ url: https://www.linkedin.com/in/charly-poly/
+ image_url: /img/people/icon/charly.jpeg
diff --git a/blog/2021-01-25-history-of-weaviate/img/vector-space.jpeg b/blog/2021-01-25-history-of-weaviate/img/vector-space.jpeg
deleted file mode 100644
index d3ce150bba..0000000000
Binary files a/blog/2021-01-25-history-of-weaviate/img/vector-space.jpeg and /dev/null differ
diff --git a/blog/2021-01-25-history-of-weaviate/img/vector-space.jpg b/blog/2021-01-25-history-of-weaviate/img/vector-space.jpg
new file mode 100644
index 0000000000..5406ef1666
Binary files /dev/null and b/blog/2021-01-25-history-of-weaviate/img/vector-space.jpg differ
diff --git a/blog/2021-01-25-history-of-weaviate/img/weaviate-console.jpeg b/blog/2021-01-25-history-of-weaviate/img/weaviate-console.jpeg
deleted file mode 100644
index c41cd66ab5..0000000000
Binary files a/blog/2021-01-25-history-of-weaviate/img/weaviate-console.jpeg and /dev/null differ
diff --git a/blog/2021-01-25-history-of-weaviate/img/weaviate-console.jpg b/blog/2021-01-25-history-of-weaviate/img/weaviate-console.jpg
new file mode 100644
index 0000000000..3fb40de26b
Binary files /dev/null and b/blog/2021-01-25-history-of-weaviate/img/weaviate-console.jpg differ
diff --git a/blog/2021-01-25-history-of-weaviate/index.mdx b/blog/2021-01-25-history-of-weaviate/index.mdx
index 7b28dd96cf..85fd5f4001 100644
--- a/blog/2021-01-25-history-of-weaviate/index.mdx
+++ b/blog/2021-01-25-history-of-weaviate/index.mdx
@@ -52,7 +52,7 @@ Finding more context around a word has to do with a concept called disambiguatio
To validate if we could disambiguate the word "apple" the following simple step was taken. What if we looked for all the words that are in the neighborhood of the space in between "apple" and "fruit"? Turns out the results are way better! We can disambiguate by moving through the hyperspace while using individual words as beacons to navigate.
-
+
In the next step, the goal was to validate if we could keep semantic meaning when storing a data object in the hyperspace by calculating the centroid using the individual words as beacons. We did that as follows, the title of this Vogue article: "Louis Vuitton's New Capsule with League of Legends Brings French High Fashion to Online Gaming — and Vice Versa".
@@ -90,7 +90,7 @@ Examples of implementations include: classification of invoices into categories,
## The Future
Weaviate will stay fully open source for the community to use. The team is growing to accelerate building Weaviate and supporting users. We are releasing new features very frequently, like new vector indexes and search pipeline features, and Weaviate Cloud and Weaviate Query App.
-
+
## Listen to Bob's story at the Open Source Data
Want to learn more about the background of Vector Search and how the ecosystem is developing? [Listen to Sam Ramji from DataStax interviewing Bob van Luijt about Vector Search and the AI stack.](https://www.datastax.com/resources/podcast/vector-search-ai-stack-more-bob-van-luijt).
diff --git a/blog/2022-06-23-the-ai-first-database-ecosystem/img/hero.jpeg b/blog/2022-06-23-the-ai-first-database-ecosystem/img/hero.jpg
similarity index 100%
rename from blog/2022-06-23-the-ai-first-database-ecosystem/img/hero.jpeg
rename to blog/2022-06-23-the-ai-first-database-ecosystem/img/hero.jpg
diff --git a/blog/2022-06-23-the-ai-first-database-ecosystem/index.mdx b/blog/2022-06-23-the-ai-first-database-ecosystem/index.mdx
index 143cdfa6bc..7ffef54845 100644
--- a/blog/2022-06-23-the-ai-first-database-ecosystem/index.mdx
+++ b/blog/2022-06-23-the-ai-first-database-ecosystem/index.mdx
@@ -4,12 +4,12 @@ slug: the-ai-first-database-ecosystem
authors: [bob]
date: 2022-06-23
tags: ['concepts']
-image: ./img/hero.jpeg
+image: ./img/hero.jpg
# canonical-url: https://www.forbes.com/sites/forbestechcouncil/2022/06/23/the-ai-first-database-ecosystem/
# canonical-name: Forbes
description: "Learn about the vision of the AI-First Database Ecosystem, which drives the R&D of the databases of the future."
---
-
+
diff --git a/blog/2022-09-07-weaviate-1-15-release/img/ordered-imports.png b/blog/2022-09-07-weaviate-1-15-release/img/ordered-imports.png
index 9d5dd92251..a1eb529eb9 100644
Binary files a/blog/2022-09-07-weaviate-1-15-release/img/ordered-imports.png and b/blog/2022-09-07-weaviate-1-15-release/img/ordered-imports.png differ
diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/img/hero.png b/blog/2022-09-13-why-is-vector-search-so-fast/img/hero.png
deleted file mode 100644
index bab4344706..0000000000
Binary files a/blog/2022-09-13-why-is-vector-search-so-fast/img/hero.png and /dev/null differ
diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx b/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx
deleted file mode 100644
index 1c338f38a7..0000000000
--- a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx
+++ /dev/null
@@ -1,130 +0,0 @@
----
-title: Why is Vector Search so fast?
-slug: why-is-vector-search-so-fast
-authors: [laura]
-date: 2022-09-13
-tags: ['search']
-image: ./img/hero.png
-description: "Vector Databases can run semantic queries on multi-million datasets in milliseconds. How is that possible?"
----
-
-
-
-
-## Why is this so incredibly fast?
-
-Whenever I talk about vector search, I like to demonstrate it with an example of a semantic search. To add the wow factor, I like to run my queries on a Wikipedia dataset, which is populated with over 28 million paragraphs sourced from Wikipedia.
-
-For example, I can query Weaviate for articles related to: "urban planning in Europe", and the vector database (in the case of my demo – [Weaviate](/developers/weaviate/)) responds with a series of articles about the topic, such as "The cities designed to be capitals".
-You can try it for yourself by following this [link](https://link.weaviate.io/3LiVxqp), which is already pre-populated with the above question. Press the play button, to see the magic happen.
-
-Here is the thing, finding the correct answer in a gigantic repository of unstructured data is not the most impressive part of this demonstration (I mean, it is very impressive), but it is the 🚀 speed at which it all happens. It takes a fraction of a second for the UI to show the results.
-
-We are talking about a semantic search query, which **takes milliseconds** to find an answer in a dataset containing **28 million paragraphs**. Interestingly enough, it takes longer to render the results than it takes the vector database to find the answer.
-
-*Note, a semantic search is unlike a regular keywords search (which matches keywords like-for-like), but instead, we are searching for answers based on the semantic meaning of our query and data.*
-
-The inevitable question that follows up this demonstration is always:
-
-> Why is this so incredibly fast?
-
-## What is a vector search?
-To answer this question we need to look at how vector databases work.
-
-Vector databases index data, unlike other databases, based on data vectors (or vector embeddings). Vector embeddings capture the meaning and context of data, usually predicted by Machine Learning models.
-
-At the time of entry/import (or any significant changes to data objects), for every new/updated data object, vector databases use Machine Learning models to predict and calculate vector embeddings, which then are stored together with the object.
-
-> Every data object in a dataset gets a vector
-
-In a nutshell, vector embeddings are an array of numbers, which can be used as coordinates in a high-dimensional space. Although it is hard to imagine coordinates in more than 3-dimensional space (x, y, z), we can still use the vectors to compute the distance between vectors, which can be used to indicate similarity between objects.
-
-There are many different distance metrics, like [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) and [Euclidean distance (L2 distance)](https://en.wikipedia.org/wiki/Euclidean_distance).
-
-### The search part
-In a similar fashion, whenever we run a query (like: "What is the tallest building in Berlin?"), a vector database can also convert it to a "query" vector. The task of a vector database is to identify and retrieve a list of vectors that are closest to the given vector of your query, using a distance metric and a search algorithm.
-
-This is a bit like a game of boules – where the small marker (jack) is the location of our query vector, and the balls (boules) are our data vectors – and we need to find the boules that are nearest to the marker.
-
-## k-nearest neighbors (kNN)
-One way to find similar vectors is with a simple [k-nearest neighbors (kNN) algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm), which returns the k nearest vectors, by comparing every data vector in the database to the query vector.
-
-In our boules example (as illustraded below), with 6 boules, the kNN algorithm would measure the distance between the jack and each of the 6 boules on the ground. This would result in 6 separate calaculations.
-
-
-
-*[Figure 1 - kNN search in a game of Boules.]*
-
-### A kNN search is computationally very expensive
-Comparing a search vector with 10, 100, or 1000 data vectors in just two dimensions is an easy job. But of course, in the real world, we are more likely to deal with millions (like in the Wikipedia dataset) or even billions of data items. In addition, the number of dimensions that most ML models use in semantic search goes up to hundreds or thousands of dimensions!
-
-The *brute* force of a **kNN search is computationally very expensive** - and depending on the size of your database, a single query could take anything from several seconds to even hours (yikes😅). If you compare a vector with 300 dimensions with 10M vectors, the vector search would need to do 300 x 10M = 3B computations! The number of required calculations increases linearly with the number of data points (O(n)) (Figure 2).
-
-
-
-*[Figure 2 - O(n) and O(log n) complexity]*
-
-In summary, kNN search doesn't scale well, and it is hard to image using it with a large dataset in production.
-
-## The answer - Approximate nearest neighbors (ANN)
-Instead of comparing vectors one by one, vector databases use [Approximate Nearest Neighbor (ANN) algorithms](https://en.wikipedia.org/wiki/Nearest_neighbor_search#Approximation_methods), which trade off a bit of accuracy (hence the A in the name) for a huge gain in speed.
-
-ANN algorithms may not return the true k nearest vectors, but they are very efficient. ANN algorithms maintain good performance (sublinear time, e.g. (poly)logarithmic complexity, see Figure 2) on very large-scale datasets.
-
-*Note that most vector databases allow you to configure how your ANN algorithm should behave. This lets you find the right balance between the **recall tradeoff** (the fraction of results that are the true top-k nearest neighbors), **latency**, **throughput** (queries per second) and **import time**.*
-*For a great example, check [Weaviate benchmarks](/developers/weaviate/benchmarks/ann#sift1m-1m-128d-vectors-l2-distance), to see how three parameters – [efConstruction, maxConnections and ef](/developers/weaviate/benchmarks/ann#what-is-being-measured) - affect recall, latency, throughput and import times.*
-
-### Examples of ANN algorithms
-Examples of ANN methods are:
-* **trees** – e.g. [ANNOY](https://github.com/spotify/annoy) (Figure 3),
-* **proximity** **graphs** - e.g. [HNSW](https://arxiv.org/abs/1603.09320) (Figure 4),
-* **clustering** - e.g. [FAISS](https://github.com/facebookresearch/faiss),
-* **hashing** - e.g. [LSH](https://en.wikipedia.org/wiki/Locality-sensitive_hashing),
-* **vector compression** - e.g. [PQ](https://ieeexplore.ieee.org/document/5432202) or [SCANN](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html).
-
-
-*[Figure 3 - Tree-based ANN search]*
-
-Which algorithm works best depends on your project. Performance can be measured in terms of latency, throughput (queries per second), build time, and accuracy (recall). These four components often have a tradeoff, so it depends on the use case which method works best.
-
-So, while ANN is not some magic method that will always find the true k nearest neighbors in a dataset, it can find a pretty good approximation of the true k neighbors. And it can do this in a fraction of the time!
-
-## HNSW in Weaviate
-[Weaviate](/developers/weaviate/) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The first ANN algorithm introduced to Weaviate is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](/developers/weaviate/concepts/vector-index#hnsw).
-
-
-
-*[Figure 4 - Proximity graph-based ANN search]*
-
-Check out [Weaviate ANN benchmarks](/developers/weaviate/benchmarks/ann) to see how HNSW performed on realistic large-scale datasets. You can use it to compare the tradeoffs between recall, QPS, latency, and import time.
-You will find it interesting to see, that Weaviate can maintain very high recall rates (>95%), whilst keeping high throughput and low latency (both in milliseconds). That is exactly what you need for fast, but reliable vector search!
-
-## Summary
-A quick recap:
-* Vector databases use Machine Learning models to calculate and attach Vector embeddings to all data objects
-* Vector embeddings capture the meaning and context of data
-* Vector databases offer super fast queries thanks to ANN algorithms
-* ANN algorithms trade a small amount of accuracy for huge gains in performance
-
-*Of course, there is a lot more going on in a vector database that makes it so efficient. But that is content for another article.😉*
-
-## Learn more
-The Weaviate Core team is currently working on research and implementation for other ANN algorithms. We are going to publish some of our findings in the next couple of weeks. So, stay tuned for more content on the topic.
-
-Until then, you can listen to a [podcast about ANN Benchmarks](https://youtu.be/kG3ji89AFyQ) with Connor and Etienne from Weaviate.
-
-import WhatsNext from '/_includes/what-next.mdx';
-
-
diff --git a/blog/2022-10-11-ann-algorithms-vamana-vs-hnsw/index.mdx b/blog/2022-10-11-ann-algorithms-vamana-vs-hnsw/index.mdx
index 626ae59ff4..411fffbc85 100644
--- a/blog/2022-10-11-ann-algorithms-vamana-vs-hnsw/index.mdx
+++ b/blog/2022-10-11-ann-algorithms-vamana-vs-hnsw/index.mdx
@@ -25,7 +25,7 @@ In this series of blog posts, we will take you on a journey with us as we resear
In this article, we will cover the need for disk-based solutions, explore Vamana (how it works and contrast it against HNSW), and present the result of Vamana implementation.
-> If you are new to vector databases and ANN, I recommend you to read "[Why is Vector Search so Fast?](/blog/why-is-vector-search-so-fast)"
+> If you are new to vector databases and ANN, I recommend you to read "[Why is Vector Search so Fast?](/blog/vector-search-explained)"
> The article explains what vector databases are and how they work.
## Need for approximation
diff --git a/blog/2022-10-18-how-to-build-an-image-search-application-with-weaviate/index.mdx b/blog/2022-10-18-how-to-build-an-image-search-application-with-weaviate/index.mdx
index 92d3636b4a..01415ae3d6 100644
--- a/blog/2022-10-18-how-to-build-an-image-search-application-with-weaviate/index.mdx
+++ b/blog/2022-10-18-how-to-build-an-image-search-application-with-weaviate/index.mdx
@@ -25,7 +25,7 @@ This blog post assumes you are familiar with vector search, as well as with spin
* [Docker Installation](https://docs.docker.com/engine/install/)
-[Check out this article](/blog/why-is-vector-search-so-fast) to learn more about why vector databases are so fast and how they work.
+[Check out this article](/blog/vector-search-explained) to learn more about why vector databases are so fast and how they work.
We will provide code snippets in this post, but you can check out the full code base in the [Weaviate Examples GitHub repo](https://github.com/weaviate/weaviate-examples) under the `nearest-neighbor-dog-search` directory. We encourage you to follow along with us by cloning the repository!
diff --git a/blog/2022-12-01-vector-library-vs-vector-database/index.mdx b/blog/2022-12-01-vector-library-vs-vector-database/index.mdx
index 498db27fdf..68e3c1714c 100644
--- a/blog/2022-12-01-vector-library-vs-vector-database/index.mdx
+++ b/blog/2022-12-01-vector-library-vs-vector-database/index.mdx
@@ -15,7 +15,7 @@ In the world of Vector Search, we use **vector embeddings** – generated by **M
We can use [vector distance functions](/blog/distance-metrics-in-vector-search) like euclidean distance or cosine distance to determine if these objects are similar to other objects in the collection. However, to do this we need to compare the distances between the query vector and **every** vector in the collection. This type of calculation can be difficult to scale to millions or billions of vectors.
-This is why we have **Vector Databases** and **Vector Libraries**. They both use the Approximate Nearest Neighbor (ANN) algorithm to search through vectors in a tiny fraction of the time. You can learn more about this topic from ["Why Vectors Search is so Fast."](/blog/why-is-vector-search-so-fast)
+This is why we have **[Vector Databases](/blog/what-is-a-vector-database/)** and **Vector Libraries**. They both use the Approximate Nearest Neighbor (ANN) algorithm to search through vectors in a tiny fraction of the time. You can learn more about this topic from ["Why Vectors Search is so Fast."](/blog/vector-search-explained)
## The Big Question
So, if both vector databases and vector libraries allow you to efficiently search through your vectors. What are the key differences between them, and why/when should you choose one over the other?
diff --git a/blog/2022-12-20-weaviate-1-17-release/img/bm25.png b/blog/2022-12-20-weaviate-1-17-release/img/bm25.png
index f41a28f890..441b55e94a 100644
Binary files a/blog/2022-12-20-weaviate-1-17-release/img/bm25.png and b/blog/2022-12-20-weaviate-1-17-release/img/bm25.png differ
diff --git a/blog/2022-12-20-weaviate-1-17-release/img/faster-startup.png b/blog/2022-12-20-weaviate-1-17-release/img/faster-startup.png
index 67a62a7c68..3102064bd3 100644
Binary files a/blog/2022-12-20-weaviate-1-17-release/img/faster-startup.png and b/blog/2022-12-20-weaviate-1-17-release/img/faster-startup.png differ
diff --git a/blog/2022-12-20-weaviate-1-17-release/img/hero.png b/blog/2022-12-20-weaviate-1-17-release/img/hero.png
index c463b81a4c..57f8e9be0a 100644
Binary files a/blog/2022-12-20-weaviate-1-17-release/img/hero.png and b/blog/2022-12-20-weaviate-1-17-release/img/hero.png differ
diff --git a/blog/2022-12-20-weaviate-1-17-release/img/hybrid-search.png b/blog/2022-12-20-weaviate-1-17-release/img/hybrid-search.png
index ab992e14ca..132721b755 100644
Binary files a/blog/2022-12-20-weaviate-1-17-release/img/hybrid-search.png and b/blog/2022-12-20-weaviate-1-17-release/img/hybrid-search.png differ
diff --git a/blog/2022-12-20-weaviate-1-17-release/img/other-improvements.png b/blog/2022-12-20-weaviate-1-17-release/img/other-improvements.png
index 67a62a7c68..3102064bd3 100644
Binary files a/blog/2022-12-20-weaviate-1-17-release/img/other-improvements.png and b/blog/2022-12-20-weaviate-1-17-release/img/other-improvements.png differ
diff --git a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/LSM-stores.png b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/LSM-stores.png
index eb69001fe2..59254170f7 100644
Binary files a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/LSM-stores.png and b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/LSM-stores.png differ
diff --git a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/batch-objects-latency.png b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/batch-objects-latency.png
index 5d6d0a1327..59779078db 100644
Binary files a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/batch-objects-latency.png and b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/batch-objects-latency.png differ
diff --git a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/gear.png b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/gear.png
index f48f1bfcb6..ef8ab14bcb 100644
Binary files a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/gear.png and b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/gear.png differ
diff --git a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/hero.png b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/hero.png
index fbecff99a0..6bed5a859d 100644
Binary files a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/hero.png and b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/hero.png differ
diff --git a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/magnifying-glass.png b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/magnifying-glass.png
index c104281515..96c28ca4ab 100644
Binary files a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/magnifying-glass.png and b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/magnifying-glass.png differ
diff --git a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/number-of-vectors-imported.png b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/number-of-vectors-imported.png
index c4d76b7000..8915d236c0 100644
Binary files a/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/number-of-vectors-imported.png and b/blog/2022-12-27-details-behind-the-sphere-dataset-in-weaviate/img/number-of-vectors-imported.png differ
diff --git a/blog/2023-01-03-hybrid-search-explained/index.mdx b/blog/2023-01-03-hybrid-search-explained/index.mdx
index d5755ddbc0..9cc00f9529 100644
--- a/blog/2023-01-03-hybrid-search-explained/index.mdx
+++ b/blog/2023-01-03-hybrid-search-explained/index.mdx
@@ -23,7 +23,7 @@ Sparse and dense vectors are calculated with distinct algorithms. Sparse vectors
Note, the current implementation of hybrid search in Weaviate uses BM25/BM25F and vector search.
-If you’re interested to learn about how dense vector indexes are built and optimized in Weaviate, check out this [article](/blog/why-is-vector-search-so-fast).
+If you’re interested to learn about how dense vector indexes are built and optimized in Weaviate, check out this [article](/blog/vector-search-explained).
### BM25
BM25 builds on the keyword scoring method [TF-IDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf) (Term-Frequency Inverse-Document Frequency) by taking the [Binary Independence Model](https://en.wikipedia.org/wiki/Binary_Independence_Model) from the IDF calculation and adding a normalization penalty that weighs a document’s length relative to the average length of all the documents in the database.
@@ -39,7 +39,7 @@ BM25F was also implemented in Weaviate `1.17`. BM25F is a variant of BM25 that a
### Dense Vectors
Dense vectors represent information stored in a database; this includes text, images, and other types of data. These embeddings are generated from machine learning models that convert data to vectors. The vectors are densely packed with information and are mostly made up of non-zero values. The meaning of each value in the vector depends on the machine learning model that you used.
-Vector databases, like [Weaviate](/developers/weaviate/), store these embeddings and calculate the distance between the two vectors. [Distance metrics](/blog/distance-metrics-in-vector-search) show how similar or dissimilar two vector embeddings are. The search query is converted to a vector, similar to the data vectors, and the distance value determines how close the vectors are.
+[Vector databases](/blog/what-is-a-vector-database/), like [Weaviate](/developers/weaviate/), store these embeddings and calculate the distance between the two vectors. [Distance metrics](/blog/distance-metrics-in-vector-search) show how similar or dissimilar two vector embeddings are. The search query is converted to a vector, similar to the data vectors, and the distance value determines how close the vectors are.
@@ -102,7 +102,7 @@ With just a few lines of code, you can start using hybrid search. You can run a
## Summary
-This article explained the concept of hybrid search. In the context of the Weaviate vector database, hybrid search is a combination of keyword search and semantic search. By combining these two types of searches, hybrid search queries can lead to more relevant results in use cases where you want to leverage the power of semantic search but also rely on exact matches of certain keywords, such as product or brand names.
+This article explained the concept of hybrid search. In the context of the Weaviate vector database, hybrid search is a combination of keyword search and semantic search. By combining these two types of searches, hybrid search queries can lead to more relevant results in use cases where you want to leverage the power of semantic search but also rely on exact matches of certain keywords, such as product or brand names.
Check out the [documentation](/developers/weaviate/api/graphql/search-operators#hybrid) for more information on hybrid search!
diff --git a/blog/2023-01-16-vector-embeddings-explained/index.mdx b/blog/2023-01-16-vector-embeddings-explained/index.mdx
index ca43f789f2..df074dc5cd 100644
--- a/blog/2023-01-16-vector-embeddings-explained/index.mdx
+++ b/blog/2023-01-16-vector-embeddings-explained/index.mdx
@@ -20,7 +20,7 @@ Semantic searches (as well as question answering) are essentially searches by si
How can computers mimic our understanding of language, and similarities of words or paragraphs? To tackle this problem, semantic search uses at its core a data structure called vector embedding (or simply, vector or embedding), which is an array of numbers. Here's how the semantic search above works, step by step:
1. The [vector database](https://weaviate.io/blog/vector-library-vs-vector-database) computes a vector embedding for each data object as it is inserted or updated into the database, using a given model.
-2. The embeddings are placed into an index, so that the database can [quickly](https://weaviate.io/blog/why-is-vector-search-so-fast) perform searches.
+2. The embeddings are placed into an index, so that the database can [quickly](https://weaviate.io/blog/vector-search-explained) perform searches.
3. For each query,
* a vector embedding is computed using the same model that was used for the data objects.
* using a special algorithm, the database finds the [closest](https://weaviate.io/blog/distance-metrics-in-vector-search) vectors to the given vector computed for the query.
@@ -80,7 +80,7 @@ If we look at each of the 5 elements of the vectors, we can see quickly that cat
But where do these numbers come from? That’s where the real magic is, and where advances in modern deep learning have made a huge impact.
## How to create vector embeddings?
-The magic of vector search resides primarily in how the embeddings are generated for each entity and the query, and secondarily in how to efficiently search within very large datasets (see our “[Why is Vector Search so Fast](https://weaviate.io/blog/why-is-vector-search-so-fast)” article for the latter).
+The magic of vector search resides primarily in how the embeddings are generated for each entity and the query, and secondarily in how to efficiently search within very large datasets (see our “[Why is Vector Search so Fast](https://weaviate.io/blog/vector-search-explained)” article for the latter).
As we mentioned, vector embeddings can be generated for various media types such as text, images, audio and others. For text, vectorization techniques have evolved tremendously over the last decade, from the venerable [word2vec (2013)](https://en.wikipedia.org/wiki/Word2vec), to the state-of-the-art transformer models era, spurred by the release of [BERT](https://en.wikipedia.org/wiki/BERT_(language_model)) in [2018](https://ai.googleblog.com/2018/11/open-sourcing-bert-state-of-art-pre.html).
### Word-level dense vector models (word2vec, GloVe, etc.)
@@ -103,7 +103,7 @@ Some of the potential downsides include:
Despite these downsides, transformer models have been wildly successful. Countless text vectorizer models have proliferated over the recent past. Plus, many more vectorizer models exist for other data types such as audio, video and images, to name a few. Some models, such as [CLIP](https://openai.com/blog/clip/), are capable of vectorizing multiple data types (images and text in this case) into one vector space, so that an image can be searched by its content using only text.
## Vector embedding visualization
-Below we can see what vector embeddings of data objects in a vector space could look like. The image shows each object embedded as a 3-dimensional vectors for ease of understanding, realistically a vector can be anywhere from ~100 to 4000 dimensions.
+Below we can see what vector embeddings of data objects in a vector space could look like. The image shows each object embedded as a 3-dimensional vectors for ease of understanding, realistically a vector can be anywhere from ~100 to 4000 dimensions.
In the following image, you can see the vectors for the words “Wolf” and “Dog” are close to each other because dogs are direct descendants of wolves. Close to the dog, you can see the word “Cat,” which is similar to the word “Dog” because both are animals that are also common pets. But further away, on the right-hand side, you can see words that represent fruit, such as “Apple” or “Banana”, which are close to each other but further away from the animal terms.
@@ -125,7 +125,7 @@ Vector embeddings capture the semantic relationship between data objects in nume
Now that you have a good understanding of what vector embeddings are and how vector embeddings are created, you might also be interested in the following articles:
* [What is a vector database?](https://weaviate.io/blog/what-is-a-vector-database)
* [Distance metrics for nearest neighbor search](https://weaviate.io/blog/distance-metrics-in-vector-search)
-* [Why is vector search so fast?](https://weaviate.io/blog/why-is-vector-search-so-fast)
+* [Why is vector search so fast?](https://weaviate.io/blog/vector-search-explained)
**Last Updated On**: August 27th, 2024
diff --git a/blog/2023-01-24-how-ai-creates-art/img/denoisingimage.jpg b/blog/2023-01-24-how-ai-creates-art/img/denoisingimage.jpg
new file mode 100644
index 0000000000..5473a667a0
Binary files /dev/null and b/blog/2023-01-24-how-ai-creates-art/img/denoisingimage.jpg differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/denoisingimage.png b/blog/2023-01-24-how-ai-creates-art/img/denoisingimage.png
deleted file mode 100644
index 3a0c332a2e..0000000000
Binary files a/blog/2023-01-24-how-ai-creates-art/img/denoisingimage.png and /dev/null differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/hero.png b/blog/2023-01-24-how-ai-creates-art/img/hero.png
index 005b8d98fa..6c10d9c151 100644
Binary files a/blog/2023-01-24-how-ai-creates-art/img/hero.png and b/blog/2023-01-24-how-ai-creates-art/img/hero.png differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/mscoco.png b/blog/2023-01-24-how-ai-creates-art/img/mscoco.png
index 287e11388b..cf4aa4f1d0 100644
Binary files a/blog/2023-01-24-how-ai-creates-art/img/mscoco.png and b/blog/2023-01-24-how-ai-creates-art/img/mscoco.png differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/noising_denoising.png b/blog/2023-01-24-how-ai-creates-art/img/noising_denoising.png
index c5ee3a5d06..248069251c 100644
Binary files a/blog/2023-01-24-how-ai-creates-art/img/noising_denoising.png and b/blog/2023-01-24-how-ai-creates-art/img/noising_denoising.png differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/noisingimage.jpg b/blog/2023-01-24-how-ai-creates-art/img/noisingimage.jpg
new file mode 100644
index 0000000000..d142fdbf5d
Binary files /dev/null and b/blog/2023-01-24-how-ai-creates-art/img/noisingimage.jpg differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/noisingimage.png b/blog/2023-01-24-how-ai-creates-art/img/noisingimage.png
deleted file mode 100644
index 8a70b9948b..0000000000
Binary files a/blog/2023-01-24-how-ai-creates-art/img/noisingimage.png and /dev/null differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/perception_of_the_world.jpg b/blog/2023-01-24-how-ai-creates-art/img/perception_of_the_world.jpg
index 324483c4f0..4183d48a49 100644
Binary files a/blog/2023-01-24-how-ai-creates-art/img/perception_of_the_world.jpg and b/blog/2023-01-24-how-ai-creates-art/img/perception_of_the_world.jpg differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/searching_through_multi_dimensional_vector_space.jpg b/blog/2023-01-24-how-ai-creates-art/img/searching_through_multi_dimensional_vector_space.jpg
index 084a3ac9ca..8bd18661b7 100644
Binary files a/blog/2023-01-24-how-ai-creates-art/img/searching_through_multi_dimensional_vector_space.jpg and b/blog/2023-01-24-how-ai-creates-art/img/searching_through_multi_dimensional_vector_space.jpg differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/the_mona_lisa_drip_painted.jpg b/blog/2023-01-24-how-ai-creates-art/img/the_mona_lisa_drip_painted.jpg
index 161dd0faf3..67dcdc633b 100644
Binary files a/blog/2023-01-24-how-ai-creates-art/img/the_mona_lisa_drip_painted.jpg and b/blog/2023-01-24-how-ai-creates-art/img/the_mona_lisa_drip_painted.jpg differ
diff --git a/blog/2023-01-24-how-ai-creates-art/img/three_images.png b/blog/2023-01-24-how-ai-creates-art/img/three_images.png
index d17ba6b3a6..846301b620 100644
Binary files a/blog/2023-01-24-how-ai-creates-art/img/three_images.png and b/blog/2023-01-24-how-ai-creates-art/img/three_images.png differ
diff --git a/blog/2023-01-24-how-ai-creates-art/index.mdx b/blog/2023-01-24-how-ai-creates-art/index.mdx
index 1e3bcd9188..af03c0d3d7 100644
--- a/blog/2023-01-24-how-ai-creates-art/index.mdx
+++ b/blog/2023-01-24-how-ai-creates-art/index.mdx
@@ -45,7 +45,7 @@ The question then becomes how diffusion models can learn (or even approximate) t
*[Source](https://yang-song.net/blog/2021/score/)*
-
+
*[Source](https://huggingface.co/blog/annotated-diffusion)*
This “noising” process, shown in the images above allows us to take training set images and add known quantities of noise to it until it becomes completely random noise. This process takes images from a state of having high probability of being found in the training set to having a low probability of existing in the training set.
@@ -55,7 +55,7 @@ Once the “noising” step is completed, then we can use these clean and noisy
*[Source](https://yang-song.net/blog/2021/score/)*
-
+
*[Source](https://huggingface.co/blog/annotated-diffusion)*
Now if we take this trained diffusion model and just give it a random static image and run the de-noising process it will transform the static image into an image that resembles images in the training set!
diff --git a/blog/2023-01-31-weaviate-podcast-search/img/andrew-huberman-search.png b/blog/2023-01-31-weaviate-podcast-search/img/andrew-huberman-search.png
index 3f74dd4439..db8fd4e277 100644
Binary files a/blog/2023-01-31-weaviate-podcast-search/img/andrew-huberman-search.png and b/blog/2023-01-31-weaviate-podcast-search/img/andrew-huberman-search.png differ
diff --git a/blog/2023-01-31-weaviate-podcast-search/img/hero.png b/blog/2023-01-31-weaviate-podcast-search/img/hero.png
index 4fad17e560..531aa86d04 100644
Binary files a/blog/2023-01-31-weaviate-podcast-search/img/hero.png and b/blog/2023-01-31-weaviate-podcast-search/img/hero.png differ
diff --git a/blog/2023-01-31-weaviate-podcast-search/img/podcast-search.png b/blog/2023-01-31-weaviate-podcast-search/img/podcast-search.png
index 7c310209bb..c27f277e4f 100644
Binary files a/blog/2023-01-31-weaviate-podcast-search/img/podcast-search.png and b/blog/2023-01-31-weaviate-podcast-search/img/podcast-search.png differ
diff --git a/blog/2023-02-21-combining-langchain-and-weaviate/img/hero.png b/blog/2023-02-21-combining-langchain-and-weaviate/img/hero.png
index c6123182a2..ce6c267713 100644
Binary files a/blog/2023-02-21-combining-langchain-and-weaviate/img/hero.png and b/blog/2023-02-21-combining-langchain-and-weaviate/img/hero.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/Ann.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/Ann.png
index 8cfb60ad51..a2110c82f3 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/Ann.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/Ann.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/hero.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/hero.png
index fcbc1b87c4..b040a9610f 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/hero.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/hero.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image10.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image10.png
index 034de0a170..dac90263fe 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image10.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image10.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image11.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image11.png
index 9e0a9aae19..2ca045bfdf 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image11.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image11.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image12.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image12.png
index c4eaf0d432..77bcc30940 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image12.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image12.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image13.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image13.png
index afff633552..2cb93fbe71 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image13.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image13.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image14.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image14.png
index 95f9075d1b..e41458f0b3 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image14.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image14.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image15.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image15.png
index 7fc2649ae0..0efe70f932 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image15.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image15.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image16.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image16.png
index 86c8423caa..3e641016ac 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image16.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image16.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image5.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image5.png
index 90f4eb8479..a57d41883a 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image5.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image5.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image6.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image6.png
index 73a2062721..1b4b48e8e3 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image6.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image6.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image7.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image7.png
index 4781621b83..c2cc7033df 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image7.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image7.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image8.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image8.png
index 64428a1320..047dc97dcc 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image8.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image8.png differ
diff --git a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image9.png b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image9.png
index f04e1f17fe..a98e62ab9e 100644
Binary files a/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image9.png and b/blog/2023-03-14-ann-algorithms-hnsw-pq/img/image9.png differ
diff --git a/blog/2023-03-23-what-are-llms/img/ChatGPT-2.jpg b/blog/2023-03-23-what-are-llms/img/ChatGPT-2.jpg
new file mode 100644
index 0000000000..028dc26228
Binary files /dev/null and b/blog/2023-03-23-what-are-llms/img/ChatGPT-2.jpg differ
diff --git a/blog/2023-03-23-what-are-llms/img/ChatGPT-2.png b/blog/2023-03-23-what-are-llms/img/ChatGPT-2.png
deleted file mode 100644
index 0d72bd4298..0000000000
Binary files a/blog/2023-03-23-what-are-llms/img/ChatGPT-2.png and /dev/null differ
diff --git a/blog/2023-03-23-what-are-llms/img/twitter1.jpeg b/blog/2023-03-23-what-are-llms/img/twitter1.jpeg
deleted file mode 100644
index ae32dd8111..0000000000
Binary files a/blog/2023-03-23-what-are-llms/img/twitter1.jpeg and /dev/null differ
diff --git a/blog/2023-03-23-what-are-llms/img/twitter1.jpg b/blog/2023-03-23-what-are-llms/img/twitter1.jpg
new file mode 100644
index 0000000000..065c7a1fbc
Binary files /dev/null and b/blog/2023-03-23-what-are-llms/img/twitter1.jpg differ
diff --git a/blog/2023-03-23-what-are-llms/img/twitter2.jpg b/blog/2023-03-23-what-are-llms/img/twitter2.jpg
new file mode 100644
index 0000000000..9491db44fc
Binary files /dev/null and b/blog/2023-03-23-what-are-llms/img/twitter2.jpg differ
diff --git a/blog/2023-03-23-what-are-llms/img/twitter2.png b/blog/2023-03-23-what-are-llms/img/twitter2.png
deleted file mode 100644
index 255642f2c5..0000000000
Binary files a/blog/2023-03-23-what-are-llms/img/twitter2.png and /dev/null differ
diff --git a/blog/2023-03-23-what-are-llms/img/wheeloffortune.jpeg b/blog/2023-03-23-what-are-llms/img/wheeloffortune.jpeg
deleted file mode 100644
index 593a24eb9c..0000000000
Binary files a/blog/2023-03-23-what-are-llms/img/wheeloffortune.jpeg and /dev/null differ
diff --git a/blog/2023-03-23-what-are-llms/img/wheeloffortune.jpg b/blog/2023-03-23-what-are-llms/img/wheeloffortune.jpg
new file mode 100644
index 0000000000..26aab4396d
Binary files /dev/null and b/blog/2023-03-23-what-are-llms/img/wheeloffortune.jpg differ
diff --git a/blog/2023-03-23-what-are-llms/index.mdx b/blog/2023-03-23-what-are-llms/index.mdx
index 106478c4fc..26845e2c86 100644
--- a/blog/2023-03-23-what-are-llms/index.mdx
+++ b/blog/2023-03-23-what-are-llms/index.mdx
@@ -22,7 +22,7 @@ In this blog we discuss:
When OpenAI launched ChatGPT at the end of 2022, more than one million people had tried the model in just a week and that trend has only continued with monthly active users for the chatbot service reaching over 100 Million, quicker than any service before, as reported by [Reuters](https://www.reuters.com/technology/chatgpt-sets-record-fastest-growing-user-base-analyst-note-2023-02-01/) and [Yahoo Finance](https://finance.yahoo.com/news/chatgpt-on-track-to-surpass-100-million-users-faster-than-tiktok-or-instagram-ubs-214423357.html?guccounter=1). Since then, OpenAI has been iteratively improving the underlying model, announcing the release of GPT (Generative Pretrained Transformer) 4.0 last week, with new and improved capabilities. It wouldn’t be hyperbole to say that Natural Language Processing (NLP) and Generative Large Language Models (LLMs) have taken the world by storm.
-
+
Though ChatGPT was not the first AI chatbot that has been released to the public, what really surprised people about this particular service was the breadth and depth of knowledge it had and its ability to articulate that knowledge with human-like responses. Aside from this, the generative aspect of this model is also quite apparent as it can hallucinate situations and dream up vivid details to fill in descriptions when prompted to do so. This gives the chatbot service somewhat of a human-like “creativity” - which is what adds a whole new dimension of utility for the service, as well as a wow factor to the user experience!
@@ -41,7 +41,7 @@ This explains why LLMs know something about basically everything but it still do
To understand how LLMs work let's consider a scenario. Imagine you are on the Wheel of Fortune game show and are tasked with completing the following phrase:
-
+
The vast majority of you, I bet, would answer: "EASY! It’s: '**ANOTHER FEATHER *I*N YO*U*R *C*A*P***'." And you would be correct! But humor me for a second while I play devil’s advocate and propose an alternative: Why not: “**ANOTHER FEATHER *O*N YO*U*R *C*A*T***”? This is a perfectly structured and complete English sentence. Why, then, did no one think of this alternative as a potential completion of the phrase?
@@ -72,7 +72,7 @@ One current limitation of LLMs, which can prevent them from attaining points 3 a
GPT 3.5 | GPT 4.0
:-------------------------:|:-------------------------:
- | 
+ | 
These LLMs are distilling the information they read from vast amounts of interactions between words. So both humans and LLMs in their unique ways build a statistical understanding of the language they are trained on.
diff --git a/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-dark.png b/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-dark.png
index 9d5d26438a..9487d811fe 100644
Binary files a/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-dark.png and b/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-dark.png differ
diff --git a/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-light.png b/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-light.png
index fae1a63b4a..d9e18244de 100644
Binary files a/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-light.png and b/blog/2023-03-28-monitoring-weaviate-in-production/img/Weaviate-monitoring-weaviate-in-prod-light.png differ
diff --git a/blog/2023-03-28-monitoring-weaviate-in-production/img/datadog-summary.png b/blog/2023-03-28-monitoring-weaviate-in-production/img/datadog-summary.png
index 3a38c7d364..94a8312e94 100644
Binary files a/blog/2023-03-28-monitoring-weaviate-in-production/img/datadog-summary.png and b/blog/2023-03-28-monitoring-weaviate-in-production/img/datadog-summary.png differ
diff --git a/blog/2023-03-28-monitoring-weaviate-in-production/img/hero.png b/blog/2023-03-28-monitoring-weaviate-in-production/img/hero.png
index 08f3da085c..4f22acc6d5 100644
Binary files a/blog/2023-03-28-monitoring-weaviate-in-production/img/hero.png and b/blog/2023-03-28-monitoring-weaviate-in-production/img/hero.png differ
diff --git a/blog/2023-03-28-monitoring-weaviate-in-production/img/query-latency.png b/blog/2023-03-28-monitoring-weaviate-in-production/img/query-latency.png
index 5ca22ecb2e..81a9c51201 100644
Binary files a/blog/2023-03-28-monitoring-weaviate-in-production/img/query-latency.png and b/blog/2023-03-28-monitoring-weaviate-in-production/img/query-latency.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-dark.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-dark.png
index 9334a949de..8f627a6aa2 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-dark.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-dark.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-light.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-light.png
index 2b37d1c249..ce975c7ba0 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-light.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/diagram-light.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/gdbtweet.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/gdbtweet.png
index 16e05379aa..70d257552e 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/gdbtweet.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/gdbtweet.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/hero.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/hero.png
index 0068ebc667..cf05396b30 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/hero.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/hero.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic1.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic1.png
index e6e98bd50c..4716d4c0ed 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic1.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic1.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic10.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic10.png
index 1c791658e2..76422df910 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic10.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic10.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic11.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic11.png
index 1d74ab11e2..ff0e22de50 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic11.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic11.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic12.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic12.png
index b2ad1ef743..875c570b0c 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic12.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic12.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic13.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic13.png
index 8156af4d06..69fad085b9 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic13.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic13.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic2.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic2.png
index 08c43822dd..a248b97a37 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic2.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic2.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic3.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic3.png
index 62c14a6347..8b07a086e6 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic3.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic3.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic4.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic4.png
index b5fba008e6..911c3e6af3 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic4.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic4.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic5.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic5.png
index 00646b165b..951b3819dc 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic5.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic5.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic6.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic6.png
index daa3e951c4..ce8cf33a12 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic6.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic6.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic7.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic7.png
index c74d601ac4..0f669125a8 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic7.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic7.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic8.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic8.png
index 5bbc049493..aef42c83f6 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic8.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic8.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic9.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic9.png
index 71c71e80e1..9c5824f558 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/pic9.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/pic9.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-dark.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-dark.png
index f7b8cea9a6..2d9b40e88d 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-dark.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-dark.png differ
diff --git a/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-light.png b/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-light.png
index a8707a7fe9..23a4aed932 100644
Binary files a/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-light.png and b/blog/2023-04-04-weaviate-retrieval-plugin/img/plugin-light.png differ
diff --git a/blog/2023-04-11-ranking-models/img/animation.png b/blog/2023-04-11-ranking-models/img/animation.png
index 05f797f05b..d09131092b 100644
Binary files a/blog/2023-04-11-ranking-models/img/animation.png and b/blog/2023-04-11-ranking-models/img/animation.png differ
diff --git a/blog/2023-04-11-ranking-models/img/hero.png b/blog/2023-04-11-ranking-models/img/hero.png
index 8879e17d23..253de549f0 100644
Binary files a/blog/2023-04-11-ranking-models/img/hero.png and b/blog/2023-04-11-ranking-models/img/hero.png differ
diff --git a/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-dark.png b/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-dark.png
index dc185e3918..11be9a305c 100644
Binary files a/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-dark.png and b/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-dark.png differ
diff --git a/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-light.png b/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-light.png
index 1d1868446f..61a210ec64 100644
Binary files a/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-light.png and b/blog/2023-04-18-autogpt-and-weaviate/img/Weaviate-auto-gpt-light.png differ
diff --git a/blog/2023-04-18-autogpt-and-weaviate/img/hero.png b/blog/2023-04-18-autogpt-and-weaviate/img/hero.png
index 949c978bb9..177cc45e80 100644
Binary files a/blog/2023-04-18-autogpt-and-weaviate/img/hero.png and b/blog/2023-04-18-autogpt-and-weaviate/img/hero.png differ
diff --git a/blog/2023-04-25-auth-in-weaviate/index.mdx b/blog/2023-04-25-auth-in-weaviate/index.mdx
index d2b13e304d..031bb59c10 100644
--- a/blog/2023-04-25-auth-in-weaviate/index.mdx
+++ b/blog/2023-04-25-auth-in-weaviate/index.mdx
@@ -15,8 +15,6 @@ description: "Videos on authentication: an overview, how to log in, how to set i
import ReactPlayer from 'react-player/lazy'
-## Overview
-
Authentication is one of those topics that we get quite a few questions about. And we can see why. It's a big, complex topic, and even within Weaviate, there are many options available which can make it seem quite confusing.
The core concept of authentication is relatively simple. When a client (e.g. a Weaviate client) sends a request to a server (e.g. a Weaviate database), it includes a "secret" that provides some assurances to Weaviate as to who that request is coming from, so that it can operate on that information.
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/areyahavingfunmrkrabs.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/areyahavingfunmrkrabs.png
index 57cb25861e..af9308148e 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/areyahavingfunmrkrabs.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/areyahavingfunmrkrabs.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-dark.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-dark.png
index 9334a949de..8f627a6aa2 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-dark.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-dark.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-light.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-light.png
index 2b37d1c249..ce975c7ba0 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-light.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/diagram-light.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/error.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/error.png
index a9f7cfd5e3..5c9236bd74 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/error.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/error.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/hackathon.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/hackathon.png
index 2f6dc86e8f..46df01211d 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/hackathon.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/hackathon.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/hero.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/hero.png
index 03d74b5827..770ea69da6 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/hero.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/hero.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/openaiapi.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/openaiapi.png
index 6175289fa7..02f5544eb3 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/openaiapi.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/openaiapi.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-dark.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-dark.png
index f7b8cea9a6..2d9b40e88d 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-dark.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-dark.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-light.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-light.png
index a8707a7fe9..23a4aed932 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-light.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/plugin-light.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/swagger.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/swagger.png
index 5c6e5a24d8..435100a31f 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/swagger.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/swagger.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/test1.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/test1.png
index 64115f004b..db04c62987 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/test1.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/test1.png differ
diff --git a/blog/2023-04-27-how-to-chatgpt-plugin/img/test2.png b/blog/2023-04-27-how-to-chatgpt-plugin/img/test2.png
index 4cdb49ab3d..c1152db1a8 100644
Binary files a/blog/2023-04-27-how-to-chatgpt-plugin/img/test2.png and b/blog/2023-04-27-how-to-chatgpt-plugin/img/test2.png differ
diff --git a/blog/2023-05-04-weaviate-1-19-release/img/datatypes.png b/blog/2023-05-04-weaviate-1-19-release/img/datatypes.png
index c319749da6..ae0cf58079 100644
Binary files a/blog/2023-05-04-weaviate-1-19-release/img/datatypes.png and b/blog/2023-05-04-weaviate-1-19-release/img/datatypes.png differ
diff --git a/blog/2023-05-04-weaviate-1-19-release/img/gencohere.png b/blog/2023-05-04-weaviate-1-19-release/img/gencohere.png
index 6d6935869a..6c39b02c37 100644
Binary files a/blog/2023-05-04-weaviate-1-19-release/img/gencohere.png and b/blog/2023-05-04-weaviate-1-19-release/img/gencohere.png differ
diff --git a/blog/2023-05-04-weaviate-1-19-release/img/groupby.png b/blog/2023-05-04-weaviate-1-19-release/img/groupby.png
index 9a1e330a29..300e1953ed 100644
Binary files a/blog/2023-05-04-weaviate-1-19-release/img/groupby.png and b/blog/2023-05-04-weaviate-1-19-release/img/groupby.png differ
diff --git a/blog/2023-05-04-weaviate-1-19-release/img/grpc.png b/blog/2023-05-04-weaviate-1-19-release/img/grpc.png
index fd8dc628dd..a442ae01aa 100644
Binary files a/blog/2023-05-04-weaviate-1-19-release/img/grpc.png and b/blog/2023-05-04-weaviate-1-19-release/img/grpc.png differ
diff --git a/blog/2023-05-04-weaviate-1-19-release/img/hero.png b/blog/2023-05-04-weaviate-1-19-release/img/hero.png
index 2cc370f3fb..8936be458a 100644
Binary files a/blog/2023-05-04-weaviate-1-19-release/img/hero.png and b/blog/2023-05-04-weaviate-1-19-release/img/hero.png differ
diff --git a/blog/2023-05-04-weaviate-1-19-release/img/tunableconsistency.png b/blog/2023-05-04-weaviate-1-19-release/img/tunableconsistency.png
index d31e417f1e..6bf4c9148c 100644
Binary files a/blog/2023-05-04-weaviate-1-19-release/img/tunableconsistency.png and b/blog/2023-05-04-weaviate-1-19-release/img/tunableconsistency.png differ
diff --git a/blog/2023-05-05-generative-feedback-loops/index.mdx b/blog/2023-05-05-generative-feedback-loops/index.mdx
index 1b268481d2..d349c27169 100644
--- a/blog/2023-05-05-generative-feedback-loops/index.mdx
+++ b/blog/2023-05-05-generative-feedback-loops/index.mdx
@@ -19,7 +19,7 @@ Enter Large Language Models (LLMs), a world changing technology that enables tra
The internet cut out the newspaper, now content can be delivered directly to you. Now we are going a step further, modifying content itself based on your personal preferences! Not only are we able to bring content with a great UX to people, but now we can generate content, in real time, based on existing information in the database to create an ever better and more personalized experience!
-In this article, we are going to illustrate the concept of **Generative Feedback Loops with LLMs**. Retrieving information from the Weaviate Vector Database to prompt Weaviate’s generative module and then vectorizing and saving these generations **back** into the database! Now we can semantically search through the generated content, as well as add symbolic metadata about it.
+In this article, we are going to illustrate the concept of **Generative Feedback Loops with LLMs**. Retrieving information from the Weaviate [Vector Database](/blog/what-is-a-vector-database/) to prompt Weaviate’s generative module and then vectorizing and saving these generations **back** into the database! Now we can semantically search through the generated content, as well as add symbolic metadata about it.
We will step into the shoes of AirBnB and implement a Generative Feedback Loop to process `Listing` objects. This shows how LLMs let us supercharge user experience by creating custom advertisements based on user preferences. We retrieve information about users from the DB, give that + information about the Listing to the LLM, and save the resulting custom advertisement back to the database.
diff --git a/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-dark.png b/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-dark.png
index 6e11e40251..d7b662562b 100644
Binary files a/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-dark.png and b/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-dark.png differ
diff --git a/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-light.png b/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-light.png
index e71bbf7f4d..a0edeb2c63 100644
Binary files a/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-light.png and b/blog/2023-05-23-pdfs-to-weaviate/img/Weaviate-ingesting-light.png differ
diff --git a/blog/2023-05-23-pdfs-to-weaviate/img/donut.png b/blog/2023-05-23-pdfs-to-weaviate/img/donut.png
index 3a413b43c2..89b9b7da76 100644
Binary files a/blog/2023-05-23-pdfs-to-weaviate/img/donut.png and b/blog/2023-05-23-pdfs-to-weaviate/img/donut.png differ
diff --git a/blog/2023-05-23-pdfs-to-weaviate/img/hero.png b/blog/2023-05-23-pdfs-to-weaviate/img/hero.png
index 7c4e5136fb..143bab76b3 100644
Binary files a/blog/2023-05-23-pdfs-to-weaviate/img/hero.png and b/blog/2023-05-23-pdfs-to-weaviate/img/hero.png differ
diff --git a/blog/2023-05-30-private-LLM/index.mdx b/blog/2023-05-30-private-LLM/index.mdx
index 10436f1490..7af74c40c3 100644
--- a/blog/2023-05-30-private-LLM/index.mdx
+++ b/blog/2023-05-30-private-LLM/index.mdx
@@ -57,7 +57,7 @@ Let’s dig deeper into locally deployed LLMs, and see how companies can more se
## Locally Running LLMs With Custom Data
-Prior to discussing how you can run OSS LLMs locally, let's discuss how you can get any LLM, local or remote, to answer prompts grounded in your custom data. The key ingredient here is to store the documents that you want to use to provide the LLM custom context, in a [vector database](https://weaviate.io/blog/what-is-a-vector-database) so that when needed the LLM can look up and retrieve the relevant documents and consume them to learn more context prior to generating a prompt. This not only provides the LLM custom data and context where it previously had none, but it also prevents hallucinations, since you help provide the LLM with relevant information, greatly reducing the chances it will make something up to fulfill the prompt. This process is known as [retrieval augmented generation (RAG)](https://arxiv.org/pdf/2005.11401.pdf) since you are augmenting the generation process of the LLM with retrieved documents from your vector database.
+Prior to discussing how you can run OSS LLMs locally, let's discuss how you can get any LLM, local or remote, to answer prompts grounded in your custom data. The key ingredient here is to store the documents that you want to use to provide the LLM custom context, in a [vector database](/blog/what-is-a-vector-database) so that when needed the LLM can look up and retrieve the relevant documents and consume them to learn more context prior to generating a prompt. This not only provides the LLM custom data and context where it previously had none, but it also prevents hallucinations, since you help provide the LLM with relevant information, greatly reducing the chances it will make something up to fulfill the prompt. This process is known as [retrieval augmented generation (RAG)](https://arxiv.org/pdf/2005.11401.pdf) since you are augmenting the generation process of the LLM with retrieved documents from your vector database.
Currently, RAG is a 4-step process illustrated in the figure below:
diff --git a/blog/2023-06-13-llms-and-search/img/hero.png b/blog/2023-06-13-llms-and-search/img/hero.png
index 3ba6b1f67f..bde0a712f4 100644
Binary files a/blog/2023-06-13-llms-and-search/img/hero.png and b/blog/2023-06-13-llms-and-search/img/hero.png differ
diff --git a/blog/2023-06-13-llms-and-search/img/qa-example.png b/blog/2023-06-13-llms-and-search/img/qa-example.png
index 9c9b91a18b..abf742fa71 100644
Binary files a/blog/2023-06-13-llms-and-search/img/qa-example.png and b/blog/2023-06-13-llms-and-search/img/qa-example.png differ
diff --git a/blog/2023-06-13-llms-and-search/img/tree-of-thought.png b/blog/2023-06-13-llms-and-search/img/tree-of-thought.png
index aaf7810288..99462daf64 100644
Binary files a/blog/2023-06-13-llms-and-search/img/tree-of-thought.png and b/blog/2023-06-13-llms-and-search/img/tree-of-thought.png differ
diff --git a/blog/2023-06-13-llms-and-search/img/wiki-example.png b/blog/2023-06-13-llms-and-search/img/wiki-example.png
index fc47996e7d..f3a46bf98f 100644
Binary files a/blog/2023-06-13-llms-and-search/img/wiki-example.png and b/blog/2023-06-13-llms-and-search/img/wiki-example.png differ
diff --git a/blog/2023-06-22-llamaindex-and-weaviate/img/hero.png b/blog/2023-06-22-llamaindex-and-weaviate/img/hero.png
index 3211bb076c..716e796795 100644
Binary files a/blog/2023-06-22-llamaindex-and-weaviate/img/hero.png and b/blog/2023-06-22-llamaindex-and-weaviate/img/hero.png differ
diff --git a/blog/2023-06-22-llamaindex-and-weaviate/index.mdx b/blog/2023-06-22-llamaindex-and-weaviate/index.mdx
index 9b9b24d02d..0086e28b88 100644
--- a/blog/2023-06-22-llamaindex-and-weaviate/index.mdx
+++ b/blog/2023-06-22-llamaindex-and-weaviate/index.mdx
@@ -74,7 +74,7 @@ nodes = parser.get_nodes_from_documents(blogs)
Here, we use the SimpleDirectoryReader to load in all documents from a given directory. We then use our `SimpleNodeParser` to chunk up the source documents into Node objects (text chunks).
-The next step is to 1) define a `WeaviateVectorStore`, and 2) build a vector index over this vector store using LlamaIndex.
+The next step is to 1) define a `WeaviateVectorStore`, and 2) build a [vector index](/developers/weaviate/concepts/vector-index) over this vector store using LlamaIndex.
```python
from llama_index.vector_stores import WeaviateVectorStore
diff --git a/blog/2023-06-27-multimodal-models/img/vectorspace.png b/blog/2023-06-27-multimodal-models/img/vectorspace.png
index 190d37259f..86eb53519d 100644
Binary files a/blog/2023-06-27-multimodal-models/img/vectorspace.png and b/blog/2023-06-27-multimodal-models/img/vectorspace.png differ
diff --git a/blog/2023-07-18-automated-testing/index.mdx b/blog/2023-07-18-automated-testing/index.mdx
index 5af3d7399e..d3f9e1ff06 100644
--- a/blog/2023-07-18-automated-testing/index.mdx
+++ b/blog/2023-07-18-automated-testing/index.mdx
@@ -53,7 +53,7 @@ Such complexity makes it important to test the application as a whole, and not j
text={TSCode}
startMarker="// START Connect"
endMarker="// Client is now ready to accept requests"
- language="js"
+ language="tsv2"
/>
@@ -131,7 +131,7 @@ If you have yet to install the required dependencies, run the following command:
text={TSCode}
startMarker="// START ConnectAndCleanup"
endMarker="// Client connected and schema cleared"
- language="ts"
+ language="tsv2"
/>
@@ -162,7 +162,7 @@ Here, the integration test will consist of checking that the class was created w
text={TSCode}
startMarker="// Create the class"
endMarker="// Class created successfully"
- language="ts"
+ language="tsv2"
/>
@@ -206,7 +206,7 @@ The test is simple; it verifies that all specified objects have been imported by
text={TSCode}
startMarker="// Import objects from the JSON file"
endMarker="// Import completed successfully"
- language="ts"
+ language="tsv2"
/>
@@ -244,7 +244,7 @@ Depending on your application and usage, you may find that the top result change
text={TSCode}
startMarker="// Run a test query"
endMarker="// Query test completed"
- language="ts"
+ language="tsv2"
/>
@@ -282,7 +282,7 @@ The code below brings together the setup and tests we've implemented so far - if
text={TSCode}
startMarker="// START Connect"
endMarker="// END all"
- language="ts"
+ language="tsv2"
/>
diff --git a/blog/2023-08-01-vector-database/img/database_architecture.png b/blog/2023-08-01-vector-database/img/database_architecture.png
new file mode 100644
index 0000000000..a91c43bd35
Binary files /dev/null and b/blog/2023-08-01-vector-database/img/database_architecture.png differ
diff --git a/blog/2023-08-01-vector-database/index.mdx b/blog/2023-08-01-vector-database/index.mdx
index 2ee019b967..b1396bc9b2 100644
--- a/blog/2023-08-01-vector-database/index.mdx
+++ b/blog/2023-08-01-vector-database/index.mdx
@@ -4,8 +4,9 @@ slug: what-is-a-vector-database
authors: [leonie, zain]
date: 2023-08-01
image: ./img/hero.png
+hide_table_of_contents: true
tags: ['concepts']
-description: "Vector databases explained by their core concepts of vector embeddings, vector search, and vector indexing"
+description: "What is a Vector Database? Explaination of core concepts, such as vector embeddings, vector search, and vector indexing"
---
@@ -21,38 +22,37 @@ If you have just recently heard the term “vector database” for the first tim
The excitement around vector databases - sometimes referred to as AI databases - is closely related to the release of ChatGPT. Since late 2022, the public has started to understand the capabilities of state-of-the-art large language models (LLMs), while developers have realized that vector databases can enhance these models further.
-This article will walk through and explain some of their core concepts, such as [vector embeddings](/blog/vector-embeddings-explained) and [vector search](/blog/why-is-vector-search-so-fast). Then, we will dive into the technical details of how distance metrics are used in vector search and how vector indexes enable efficient retrieval. Once we have a fundamental understanding, we will discuss vector database use cases and the current tool landscape.
+This article will explain some of their core concepts, such as [vector embeddings](/blog/vector-embeddings-explained) and [vector search](/blog/vector-search-explained). Then, we will dive into the technical details of using distance metrics in vector search and how [vector indexes](/developers/weaviate/concepts/vector-index) enable efficient retrieval. Once we have a fundamental understanding, we will discuss use cases and the current tool landscape.
## What is a Vector Database?
A vector database indexes, stores, and provides access to structured or unstructured data (e.g., text or images) alongside its vector embeddings, which are the data's numerical representation. It allows users to find and retrieve similar objects quickly at scale in production.
-Because of its search capabilities, a vector database is sometimes also called a vector search engine.
-
-This section introduces the core concepts of vector databases. We will discuss what vector embeddings are, how they enable similarity search, and how they accelerate it.
+Because of its search capabilities, it is sometimes also called a vector search engine.
## How Do Vector Databases Work?
-Imagine all the files on your computer, you could have text documents, images, songs, videos etc. A vector database can take each file and, using machine learning models, represent/embed them as vectors. In this translation from, human understandable, text, images and more into, machine understandable, vectors it tries to preserve as much of the meaning and semantics of the data as possible.
+Vector databases retrieve data objects with vector search. Vector search uses vector embeddings, which are machine-understandable formats of human-understandable data objects, such as text documents, images, songs, videos, and so on. Vector databases also use vector indexing to retrieve data objects at scale.
+
+This section introduces these core concepts. We will discuss what vector embeddings are, how they enable similarity search, and how they accelerate it.
-Once we have all the data embedded into vector representations we can now perform vector search. Let's say you want to search for photos from your vacation, so you type “family vacation in Prague” into the navigator search bar, a vector database would turn that text query into a vector and then search for the object vectors in closest proximity to the query vector. These could be text documents describing the vacation or even pictures you took and those files are then returned to you. Let’s unpack the concept of vector embeddings and vector search further below!
### Vector Embeddings
-When you think of data, you probably think of neatly organized numbers in spreadsheets. This type of data is called structured data because it can easily be stored in tabular format. But roughly 80% of today’s data is said to be unstructured. Examples of unstructured data are images, text (e.g., documents, social media posts, or emails), or time series data (e.g., audio files, sensor data, or video). The problem with this type of data is that it is difficult to store it in an organized way so you can easily find what you are looking for.
+When you think of data, you probably think of neatly organized numbers in spreadsheets. This type of data is called structured data because it can easily be stored in tabular format. However, roughly 80% of today’s data is said to be unstructured. Examples of unstructured data are images, text (e.g., documents, social media posts, or emails), or time series data (e.g., audio files, sensor data, or video). The problem with this type of data is that it is difficult to store it in an organized way so you can easily find what you are looking for.
-But innovations in Artificial Intelligence (AI) and Machine Learning have enabled us to numerically represent unstructured data without losing its semantic meaning in so-called vector embeddings. A vector embedding is just a long list of numbers, each describing a feature of the data object.
+But innovations in Artificial Intelligence (AI) and Machine Learning (ML) have enabled us to numerically represent unstructured data without losing its semantic meaning in so-called vector embeddings. A vector embedding is just a long list of numbers, each describing a feature of the data object.
An example is how we numerically represent colors [in the RGB system](https://huggingface.co/spaces/jphwang/colorful_vectors), where each number in the vector describes how red, green, or blue a color is. E.g., the following green color can be represented as [6, 205, 0] in the RGB system.
-But fitting more complex data, such as words, sentences, or text, into a meaningful series of numbers isn’t trivial. This is where Machine Learning models come in: Machine Learning models enable us to represent the contextual meaning of, e.g., a word as a vector because they have learned to represent the relationship between different words in a vector space. These types of Machine Learning models that can generate embeddings from unstructured data are also called embedding model or vectorizer.
+But fitting more complex data, such as words, sentences, or text, into a meaningful series of numbers isn’t trivial. This is where AI models come in: AI models enable us to represent the contextual meaning of, e.g., a word as a vector because they have learned to represent the relationship between different words in a vector space. AI models that can generate embeddings from unstructured data are also called embedding models or vectorizers.
Below, you can see an example of the three-dimensional vector space of the RGB system with a few sample data points (colors).
-Depending on the used embedding model, the data can be represented in different vector spaces, which can differ in the number of dimensions, ranging from tens to thousands, where the precision of the representation increases with increasing dimensions. In the color example, there are also different ways to represent a color numerically. E.g., you can represent the previous green color in the RGB system as [6, 205, 0] or in the CMYK system as [97, 0, 100, 20].
+Different embedding models represent data in different vector spaces. Vector spaces can differ in the number of dimensions, ranging from tens to thousands, where the precision of the representation increases with increasing dimensions. In the color example, there are also different ways to represent a color numerically. For example, you can represent the previous green color in the RGB system as [6, 205, 0] or in the CMYK system as [97, 0, 100, 20].
This is why it is important to use the same embedding model for all your data to ensure it is in the respective vector space.
@@ -63,7 +63,7 @@ For now, let’s consider a simpler example with numerical representations of wo
### Vector Search
-Vector embeddings allow us to find and retrieve similar objects from the vector database by searching for objects that are close to each other in the vector space, which is called vector search, similarity search, or semantic search.
+Vector embeddings allow us to find and retrieve similar objects by searching for objects that are close to each other in the vector space. This concept is called [vector search](/blog/vector-search-explained), similarity search, or semantic search.
Similarly to how we can find similar vectors for the word "Dog", we can find similar vectors to a search query. For example, to find words similar to the word “Kitten”, we can generate a vector embedding for this query term - also called a query vector - and retrieve all its nearest neighbors, such as the word “Cat”, as illustrated below.
@@ -71,32 +71,30 @@ Similarly to how we can find similar vectors for the word "Dog", we can find sim
As the concept of semantic search is based on the contextual meaning, it allows for a more human-like search experience by retrieving relevant search results that match the user's intent. This advantage makes vector search important for applications, that are e.g., sensitive to typos or synonyms.
-The numerical representation of a data object allows us to apply mathematical operations to them, such as calculating the distance between two vector representations to determine their similarity. To calculate the distance between two vectors, you can use several similarity measures. E.g., Weaviate supports the following distance metrics:
+The numerical representation of a data object allows us to apply mathematical operations to them. For example you can calculate the distance between two vector representations to determine their similarity. You can use several similarity measures to calculate the distance between two vectors. E.g., Weaviate supports the following distance metrics:
* **Squared Euclidean or L2-squared distance** calculates the straight-line distance between two vectors. Its range is [0, ∞], where 0 represents identical vectors, and larger values represent increasingly dissimilar vectors.
* **Manhattan or L1 distance** calculates the sum of the lengths of the projections of the line segment between the points onto the coordinate axes. Its range is [0, ∞], where 0 represents identical vectors, and larger values represent increasingly dissimilar vectors.
-* **Cosine similarity** calculates the cosine of the angle between two vectors. In Weaviate, the cosine distance is used for the complement of cosine similarity. Its range is [0, 2], where 0 represents identical vectors, and 2 represents vectors that point in opposite directions.
-* **Dot product** calculates the product of the magnitudes of two vectors and the cosine of the angle between them. Its range is [-∞, ∞], where 0 represents orthogonal vectors, and larger values represent increasingly similar vectors. In Weaviate, the negative dot product is used to keep the intuition that larger values represent increasingly dissimilar vectors.
-* Hamming distance calculates the number of differences between vectors at each dimension.
+* **Cosine similarity** calculates the cosine of the angle between two vectors. Weaviate uses the cosine distance for the complement of cosine similarity. Its range is [0, 2], where 0 represents identical vectors, and 2 represents vectors that point in opposite directions.
+* **Dot product** calculates the product of the magnitudes of two vectors and the cosine of the angle between them. Its range is [-∞, ∞], where 0 represents orthogonal vectors, and larger values represent increasingly similar vectors. Weaviate uses the negative dot product to keep the intuition that larger values represent increasingly dissimilar vectors.
+* **Hamming distance** calculates the number of differences between vectors at each dimension.
-As you can see, there are many different similarity measures. As a rule of thumb, select the same metric as the one used to train the Machine Learning model. If you are unsure which one was used, you can usually find this information on the hosting service’s site, e.g., [OpenAI’s ada-002 uses cosine similarity](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use) or [MiniLM hosted on Hugging Face supports cosine similarity, dot product, and Euclidean distance](https://huggingface.co/sentence-transformers/multi-qa-MiniLM-L6-cos-v1#technical-details).
+As you can see, there are many different similarity measures. As a rule of thumb, select the same metric as the one used to train the ML model. Suppose you are unsure which one was used. In that case, you can usually find this information on the hosting service’s site, e.g., [OpenAI’s ada-002 uses cosine similarity](https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use) or [MiniLM hosted on Hugging Face supports cosine similarity, dot product, and Euclidean distance](https://huggingface.co/sentence-transformers/multi-qa-MiniLM-L6-cos-v1#technical-details).
To learn more about the different distance metrics, you can continue reading our blog post on [What are Distance Metrics in Vector Search?](/blog/distance-metrics-in-vector-search)
-## Vector Database Architecture
-
### Vector Indexing for Approximate Nearest Neighbor Approach
-Vector indexing is the process of organizing vector embeddings in a way that data can be retrieved efficiently.
+Vector indexing is the process of organizing vector embeddings so that data can be retrieved efficiently.
-When you want to find the closest items to a given query vector, the brute force approach would be to use the k-Nearest Neighbors (kNN) algorithm. But calculating the similarity between your query vector and every entry in the vector database requires a lot of computational resources, especially if you have large datasets with millions or even billions of data points, because the required calculations increase linearly (O(n)) with the dimensionality and the number of data points.
+When you want to find the closest items to a given query vector, the brute force approach would be to use the k-Nearest Neighbors (kNN) algorithm. But calculating the similarity between your query vector and every entry in the vector database requires a lot of computational resources, especially if you have large datasets with millions or even billions of data points. This is because the required calculations increase linearly (O(n)) with the dimensionality and the number of data points.
-A more efficient solution to find similar objects is to use an **approximate nearest neighbor (ANN) approach**. The underlying idea is to pre-calculate the distances between the vector embeddings and organize and store similar vectors close to each other (e.g., in clusters or a graph), so that you can later find similar objects faster. This process is called vector indexing. Note that the speed gains are traded in for some accuracy because the ANN approach returns only the approximate results.
+A more efficient solution to find similar objects is to use an **approximate nearest neighbor (ANN) approach**. The underlying idea is to pre-calculate the distances between the vector embeddings. Then you can organize and store similar vectors close to each other (e.g., in clusters or a graph), so that you can later find similar objects faster. This process is called vector indexing. Note that the speed gains are traded in for some accuracy because the ANN approach returns only the approximate results.
-In the previous example, one option would be to pre-calculate some clusters, e.g., animals, fruits, and so on. So, when you query the vector database for "Kitten", you could start your search by only looking at the closest animals and not waste time calculating the distances between all fruits and other non-animal objects. More specifically, the ANN algorithm can help you to start the search in a region near, e.g., four-legged animals. Then, the algorithm would also prevent you from venturing further from relevant results, as the objects are pre-organized by similarity.
+In the previous example, one option would be to pre-calculate some clusters, e.g., animals, fruits, and so on. So, when you query the database for "Kitten", you could start your search by only looking at the closest animals and not waste time calculating the distances between all fruits and other non-animal objects. More specifically, the ANN algorithm can help you to start the search in a region near, e.g., four-legged animals. Then, the algorithm would also prevent you from venturing further from relevant results.
-The above example roughly describes the concept of the Hierarchical Navigable Small World (HNSW) algorithm, which is the default ANN algorithm in Weaviate. However, there are several ANN algorithms to index the vectors, which can be categorized into the following groups:
+The above example roughly describes the Hierarchical Navigable Small World (HNSW) algorithm. It is also the default ANN algorithm in Weaviate. However, there are several ANN algorithms to index the vectors, which can be categorized into the following groups:
* Clustering-based index (e.g., [FAISS](https://github.com/facebookresearch/faiss))
* Proximity graph-based index (e.g., [HNSW](https://arxiv.org/abs/1603.09320))
* Tree-based index (e.g., [ANNOY](https://github.com/spotify/annoy))
@@ -105,16 +103,25 @@ The above example roughly describes the concept of the Hierarchical Navigable Sm
Note that indexing enables fast retrieval at query time, but it can take a lot of time to build the index initially.
-Further reading: [Why Is Vector Search So Fast](/blog/why-is-vector-search-so-fast)
+Further reading: [Why Is Vector Search So Fast](/blog/vector-search-explained)
+
+## Vector Database Architecture
+
+Since vector databases do not only store vector embeddings but also the original data objects, their architecture consists of multiple indices.
+
+Below you can see an example of the [Weaviate vector database architecture](/developers/weaviate/concepts#weaviate-architecture).
+
+
+Users can create any number of indices and each index can contain any number of shards. Shards can be dsitributed and/or replicated across nodes in the cluster. A shard always contains object, inverted, and vector storage. Vector storage is not affected by LSM segmentation.
-## Vector Database Use Cases
-Vector databases’ search capabilities can be used in various applications ranging from classical Machine Learning use cases, such as natural language processing, computer vision, and recommender systems, to providing long-term memory to LLMs in modern applications.
+## Vector Database Use Cases
+A wide variety of applications use the search capabilities of vector databases. They range from classical ML use cases, such as natural language processing (NLP), computer vision, and recommender systems, to providing long-term memory to LLMs in modern applications.
-The **most popular use case of vector search engines is naturally for search**. Because a vector database can help find similar objects, it is predestined for applications where you might want to find similar products, movies, books, songs, etc. That’s why vector search engines are also **used in recommendation systems** as a restated task of search.
+The **most popular use case of vector databases is naturally for search**. Because a vector search can help find similar objects, it is predestined for applications where you might want to find similar products, movies, books, songs, etc. That’s why they are also **used in recommendation systems** as a restated search task.
-With the rise of LLMs, **vector databases have already been used to enhance modern Generative AI applications**.You might have already come across the term that vector databases provide LLMs with long-term memory. LLMs are stateless, which means that they immediately forget what you have just discussed if you don’t store this information in, e.g., a vector database and thus provide them with a state. This enables LLMs to hold an actual conversation. Also, you can store the domain-specific context in a vector database to minimize the LLM from hallucinating. Other popular examples of vector databases improving the capabilities of LLMs are question-answering and retrieval-augmented generation.
+With the rise of LLMs, **many modern Generative AI applications use vector databases as an external knowledge source**. You might have already come across the term that they provide LLMs with long-term memory. LLMs are stateless, which means that they immediately forget what you have just discussed if you don’t store this information in, e.g., a vector database and thus provide them with a state. This enables LLMs to hold an actual conversation. Also, you can store additional information in them as part of a[retrieval-augmented generation (RAG)](/blog/introduction-to-rag) pipeline to reduce hallucinations.
-Here is an example [demo](https://github.com/weaviate/healthsearch-demo) called [HealthSearch](https://weaviate.io/blog/healthsearch-demo) of a [Weaviate](http://weaviate.io/) vector database in action together with an LLM showcasing the potential of leveraging user-written reviews and queries to retrieve supplement products based on specific health effects.
+Here is an example [demo](https://github.com/weaviate/healthsearch-demo) called [HealthSearch](https://weaviate.io/blog/healthsearch-demo) of a [Weaviate](http://weaviate.io/) vector database in action together with an LLM showcasing the potential of leveraging user-written reviews and queries to retrieve supplement products based on specific health effects.
@@ -122,15 +129,15 @@ Here is an example [demo](https://github.com/weaviate/healthsearch-demo) called
Although vector databases are AI-native and specifically designed to handle vector embeddings and enable efficient vector search, alternatives like [vector libraries](https://weaviate.io/blog/vector-library-vs-vector-database) and vector-capable databases exist as well.
### Vector Database vs. Traditional (Relational) Database
-The main difference between a traditional (relational) database and a modern vector database comes from the type of data they were optimized for. While a relational database is designed to store structured data in columns, a vector database is also optimized to store unstructured data (e.g., text, images, or audio) and their vector embeddings.
+The main difference between a modern vector and a traditional (relational) database comes from the type of data they were optimized for. While a relational database is designed to store structured data in columns, a vector database is also optimized to store unstructured data (e.g., text, images, or audio) and their vector embeddings.
-Because vector databases and relational databases are optimized for different types of data, they also differ in the way data is stored and retrieved. In a relational database, data is stored in columns and retrieved by keyword matches in traditional search. In contrast, vector databases also store the vector embeddings of the original data, which enables efficient semantic search. Because vector search is capable of semantically understanding your search terms, it doesn't rely on retrieving relevant search results based on exact matches, which makes it robust to typos and synonyms.
+Because vector and relational databases are optimized for different types of data, they also differ in how data is stored and retrieved. In a relational database, data is stored in columns and retrieved by keyword matches in traditional search. In contrast, vector databases also store the original data's vector embeddings, enabling efficient semantic search. Because vector search can semantically understand your search terms, it doesn't rely on retrieving relevant search results based on exact matches. This makes it robust to synonyms.
For example, imagine you have a database that stores Jeopardy questions, and you want to retrieve all questions that contain an animal. Because search in traditional databases relies on keyword matches, you would have to create a big query that queries all animals (e.g., contains "dog", or contains "cat", or contains "wolf", etc.). With semantic search, you could simply query for the concept of "animals".
-Because most vector databases do not only store vector embeddings but store them together with the original source data, they are not only capable of vector search but also enable traditional keyword search. Some vector databases like Weaviate even support [hybrid search](https://weaviate.io/blog/hybrid-search-explained) capabilities that combine vector search with keyword search.
+Because most vector databases do not only store vector embeddings but store them together with the original source data, they are not only capable of vector search but also enable traditional keyword search. Some, like Weaviate, even support [hybrid search](https://weaviate.io/blog/hybrid-search-explained) capabilities that combine vector search with keyword search.
### Vector Database vs. Vector-Capable Database (SQL and NoSQL)
Today, many existing databases have already enabled vector support and vector search. However, they usually don’t index the vector embeddings, which makes the vector search slow. Thus, an advantage of AI-native vector databases over vector-capable databases is their efficiency in vector search due to vector indexing.
@@ -148,7 +155,7 @@ A graph database stores data in nodes and edges, representing entities and their
### Vector Database vs. Vector Store
-A vector store is a more lightweight system or library designed specifically to store and retrieve vector embeddings and it focuses more on managing vectors and supports basic operations like indexing and querying. A vector database on the other hand not only stores and retrieves vector embeddings but also offers comprehensive database functionalities such as distributed storage, scaling, security, and support for complex queries.
+A vector store is a lighter system or library designed specifically to store and retrieve vector embeddings. It focuses more on managing vectors and supports basic operations like indexing and querying. A vector database, on the other hand, not only stores and retrieves vector embeddings but also offers comprehensive database functionalities such as distributed storage, scaling, security, and support for complex queries.
## Vector Database FAQs
@@ -156,27 +163,30 @@ This section answers some common questions about vector databases.
### Why we need vector databases
-Vector databases are essential for efficiently managing and searching high-dimensional vector embeddings, enabling real-time accurate similarity searches that perform a critical function in the AI-native app stack.
+Vector databases are essential for efficiently managing and searching high-dimensional vector embeddings. They enable real-time accurate similarity searches, which are critical in the AI-native app stack.
### Why Use a Vector Database?
-Vector databases have become a critical component in the AI-native app stack, particularly for managing the vast amounts of unstructured data that organizations generate, such as images, text, and audio. Unlike traditional databases, which struggle with unstructured data, vector databases efficiently store, search, and analyze this data by leveraging high-dimensional vector representations, or embeddings. This capability is particularly beneficial where the ability to retrieve relevant information quickly and accurately is critical. For example, in recommendation systems or natural language processing (NLP) tasks, vector databases enable businesses to unlock valuable insights, leading to more informed decision-making and improved personalization. Furthermore, vector databases enable human-like semantic search capabilities, allowing businesses to find products, content, or information that closely matches a user's query. This not only improves user experience and engagement but also drives customer specific personalization.
+Vector databases have become a critical component in the AI-native app stack, particularly for managing the vast amounts of unstructured data that organizations generate, such as images, text, and audio. Unlike traditional databases, which struggle with unstructured data, vector databases efficiently store, search, and analyze this data by leveraging high-dimensional vector representations or embeddings. This capability is particularly beneficial where the ability to retrieve relevant information quickly and accurately is critical.
-The scalability and performance offered by vector databases are crucial as businesses grow and data volumes increase. These databases are optimized for handling large-scale, high-dimensional vectors, ensuring that data retrieval remains fast and accurate even at scale. This scalability is vital for AI applications that require real-time data processing, such as chatbots, fraud detection, and other interactive AI systems. Additionally, vector databases integrate seamlessly with existing AI workflows, including LLM orchestration frameworks, embedding model providers, and inference engines. This integration simplifies the deployment and scaling of production-level retrieval-augmented generation (RAG) applications, making it easier for businesses to harness the full potential of AI. By providing a specialized infrastructure for storing and retrieving vectors, vector databases reduce the complexity and cost associated with AI development, allowing organizations to focus on innovation and delivering personalized experiences to their customers.
+For example, in recommendation systems or NLP tasks, vector databases enable businesses to unlock valuable insights, leading to more informed decision-making and improved personalization. Furthermore, they enable human-like semantic search capabilities, allowing businesses to find products, content, or information that closely match a user's query. This not only improves user experience and engagement but also drives customer-specific personalization.
+
+Vector databases' scalability and performance are crucial as businesses grow and data volumes increase. These databases are optimized for handling large-scale, high-dimensional vectors, ensuring that data retrieval remains fast and accurate even at scale. This scalability is vital for AI applications that require real-time data processing, such as chatbots, fraud detection, and other interactive AI systems.
+
+Additionally, vector databases integrate seamlessly with existing AI workflows, including LLM orchestration frameworks, embedding model providers, and inference engines. This integration simplifies the deployment and scaling of production-level RAG applications, making it easier for businesses to harness AI's full potential. By providing a specialized infrastructure for storing and retrieving vectors, vector databases reduce the complexity and cost associated with AI development, allowing organizations to focus on innovation and delivering personalized experiences to their customers.
### How to use a vector database
-Store vector embeddings in the database, index them for efficient retrieval, and perform similarity searches to find the most relevant data points.
+Store vector embeddings in the database, index them for efficient retrieval and perform similarity searches to find the most relevant data points.
### How to choose a vector database
-
-A vector database should be chosen by assessing latency of searches, memory footprint as object count scales up, recall and retrieval relevancy, cost efficiency and integration capabilities with your existing stack and AI tools such as language models and embedding models.
+Selection criteria for vector databases are search latency, memory footprint as object count scales up, recall and retrieval relevancy, cost efficiency, and integration capabilities with your existing stack and AI tools, such as language models and embedding models.
## Summary
This article explained that vector databases are a type of database that indexes, stores, and provides access to structured or unstructured data alongside its vector embeddings. Vector databases like [Weaviate](http://weaviate.io/) allow for efficient similarity search and retrieval of data based on their vector distance or vector similarity at scale.
-We covered core concepts around vector databases, such as vector embeddings, and discussed that vector databases enable efficient vector search by leveraging ANN algorithms. Additionally, we explored other vector database tools and discussed the advantages of vector databases over traditional and vector-capable databases and vector libraries.
+We covered their core concepts, such as vector embeddings, and discussed that they enable efficient vector search by leveraging ANN algorithms. Additionally, we explored other vector search tools and discussed the advantages of vector databases over traditional and vector-capable databases and vector libraries.
-**Last Updated On**: September 18th, 2024
+**Last Updated On**: November 189th, 2024
import WhatNext from '/_includes/what-next.mdx'
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine-example.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine-example.png
index 6cbfdfa58e..4d6642fcad 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine-example.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine-example.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine.png
index e92d8d61ab..bbc4e00fbd 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/cosine.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/direction.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/direction.png
index 27795da2d7..ce85f544ff 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/direction.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/direction.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/dot-product.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/dot-product.png
index f3747f89e9..7fb839b7dc 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/dot-product.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/dot-product.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/equation1.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/equation1.png
index 6721b2d50b..10d04aceb2 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/equation1.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/equation1.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/hamming-and-bpr.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/hamming-and-bpr.png
index 6bb99a909d..9e7b92c0b6 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/hamming-and-bpr.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/hamming-and-bpr.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/hero.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/hero.png
index c23fc858e3..cdd36787be 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/hero.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/hero.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-equation.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-equation.png
index e1a0edf190..6e109b2589 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-equation.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-equation.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-formula.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-formula.png
index a1ff6005f4..67f00b8d11 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-formula.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/l2-formula.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan-example.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan-example.png
index 09585be165..d0d4090f40 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan-example.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan-example.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan.png
index ec6951bbdd..29599bdf91 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/manhattan.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/img/vectors-example.png b/blog/2023-08-15-distance-metrics-in-vector-search/img/vectors-example.png
index 8fa705dd60..69d65c5b1d 100644
Binary files a/blog/2023-08-15-distance-metrics-in-vector-search/img/vectors-example.png and b/blog/2023-08-15-distance-metrics-in-vector-search/img/vectors-example.png differ
diff --git a/blog/2023-08-15-distance-metrics-in-vector-search/index.mdx b/blog/2023-08-15-distance-metrics-in-vector-search/index.mdx
index 9d820b9e8f..6ab7308b6e 100644
--- a/blog/2023-08-15-distance-metrics-in-vector-search/index.mdx
+++ b/blog/2023-08-15-distance-metrics-in-vector-search/index.mdx
@@ -50,7 +50,7 @@ The best thing is, vector databases can **query large datasets**, containing *
Without getting too much into details, one of the big reasons why vector databases are so fast is because they use the **Approximate Nearest Neighbor** (ANN) algorithm to index data based on vectors. ANN algorithms organize indexes so that the vectors that are closely related are stored next to each other.
-Check out this article to learn ["Why is Vector Search are so Fast"](/blog/why-is-vector-search-so-fast) and how vector databases work.
+Check out this article to learn ["Why is Vector Search are so Fast"](/blog/vector-search-explained) and how vector databases work.
### Why are there Different Distance Metrics?
diff --git a/blog/2023-09-11-weaviate-gorilla-part-1/img/1.png b/blog/2023-09-11-weaviate-gorilla-part-1/img/1.png
index 9907fdcf07..927be89e36 100644
Binary files a/blog/2023-09-11-weaviate-gorilla-part-1/img/1.png and b/blog/2023-09-11-weaviate-gorilla-part-1/img/1.png differ
diff --git a/blog/2023-09-11-weaviate-gorilla-part-1/img/2.png b/blog/2023-09-11-weaviate-gorilla-part-1/img/2.png
index 6809d323ea..7e751e0220 100644
Binary files a/blog/2023-09-11-weaviate-gorilla-part-1/img/2.png and b/blog/2023-09-11-weaviate-gorilla-part-1/img/2.png differ
diff --git a/blog/2023-09-11-weaviate-gorilla-part-1/img/3.png b/blog/2023-09-11-weaviate-gorilla-part-1/img/3.png
index cdaefc08e9..bbb9856fc5 100644
Binary files a/blog/2023-09-11-weaviate-gorilla-part-1/img/3.png and b/blog/2023-09-11-weaviate-gorilla-part-1/img/3.png differ
diff --git a/blog/2023-09-11-weaviate-gorilla-part-1/img/4.png b/blog/2023-09-11-weaviate-gorilla-part-1/img/4.png
index 26319013f3..9eced34fc6 100644
Binary files a/blog/2023-09-11-weaviate-gorilla-part-1/img/4.png and b/blog/2023-09-11-weaviate-gorilla-part-1/img/4.png differ
diff --git a/blog/2023-09-11-weaviate-gorilla-part-1/img/5.png b/blog/2023-09-11-weaviate-gorilla-part-1/img/5.png
index dd1ac12471..77ffc0b1eb 100644
Binary files a/blog/2023-09-11-weaviate-gorilla-part-1/img/5.png and b/blog/2023-09-11-weaviate-gorilla-part-1/img/5.png differ
diff --git a/blog/2023-09-11-weaviate-gorilla-part-1/img/hero.png b/blog/2023-09-11-weaviate-gorilla-part-1/img/hero.png
index 4b946c96e1..b2104414fd 100644
Binary files a/blog/2023-09-11-weaviate-gorilla-part-1/img/hero.png and b/blog/2023-09-11-weaviate-gorilla-part-1/img/hero.png differ
diff --git a/blog/2023-09-26-collections-python-client/img/hero.png b/blog/2023-09-26-collections-python-client/img/hero.png
index b3cc201320..56031dde65 100644
Binary files a/blog/2023-09-26-collections-python-client/img/hero.png and b/blog/2023-09-26-collections-python-client/img/hero.png differ
diff --git a/blog/2023-09-26-collections-python-client/img/inverted-index-hints.png b/blog/2023-09-26-collections-python-client/img/inverted-index-hints.png
index d41af10189..66bfee60b9 100644
Binary files a/blog/2023-09-26-collections-python-client/img/inverted-index-hints.png and b/blog/2023-09-26-collections-python-client/img/inverted-index-hints.png differ
diff --git a/blog/2023-10-02-hacktoberfest-2023/img/hacktoberfest_2023_demo.mp4 b/blog/2023-10-02-hacktoberfest-2023/img/hacktoberfest_2023_demo.mp4
index 7a04c39e72..4c1abe66a8 100644
Binary files a/blog/2023-10-02-hacktoberfest-2023/img/hacktoberfest_2023_demo.mp4 and b/blog/2023-10-02-hacktoberfest-2023/img/hacktoberfest_2023_demo.mp4 differ
diff --git a/blog/2023-10-02-hacktoberfest-2023/img/weaviate-demos.png b/blog/2023-10-02-hacktoberfest-2023/img/weaviate-demos.png
index 693c6ee6a2..c04eeb5ee3 100644
Binary files a/blog/2023-10-02-hacktoberfest-2023/img/weaviate-demos.png and b/blog/2023-10-02-hacktoberfest-2023/img/weaviate-demos.png differ
diff --git a/blog/2023-10-02-hacktoberfest-2023/img/weaviate-hacktoberfest-2023.png b/blog/2023-10-02-hacktoberfest-2023/img/weaviate-hacktoberfest-2023.png
index 0d37229442..41c6109cec 100644
Binary files a/blog/2023-10-02-hacktoberfest-2023/img/weaviate-hacktoberfest-2023.png and b/blog/2023-10-02-hacktoberfest-2023/img/weaviate-hacktoberfest-2023.png differ
diff --git a/blog/2023-11-15-moonsift-story/img/hero.png b/blog/2023-11-15-moonsift-story/img/hero.png
index 9916177a62..ffec81d48b 100644
Binary files a/blog/2023-11-15-moonsift-story/img/hero.png and b/blog/2023-11-15-moonsift-story/img/hero.png differ
diff --git a/blog/2023-11-15-moonsift-story/img/image1.png b/blog/2023-11-15-moonsift-story/img/image1.png
index 9cdc04e0ce..3130b772ab 100644
Binary files a/blog/2023-11-15-moonsift-story/img/image1.png and b/blog/2023-11-15-moonsift-story/img/image1.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/agents.png b/blog/2023-11-21-rag-evaluation/img/agents.png
index 5134ec76ed..7b0491628b 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/agents.png and b/blog/2023-11-21-rag-evaluation/img/agents.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/chunk.png b/blog/2023-11-21-rag-evaluation/img/chunk.png
index 60bb5e42fa..53c751b81a 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/chunk.png and b/blog/2023-11-21-rag-evaluation/img/chunk.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/hero.png b/blog/2023-11-21-rag-evaluation/img/hero.png
index bbc491bd42..9404e8b6b3 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/hero.png and b/blog/2023-11-21-rag-evaluation/img/hero.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/multi-index.png b/blog/2023-11-21-rag-evaluation/img/multi-index.png
index e87fce7814..55928ce1b9 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/multi-index.png and b/blog/2023-11-21-rag-evaluation/img/multi-index.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/rag-eval.png b/blog/2023-11-21-rag-evaluation/img/rag-eval.png
index 336fbb993f..7b1da63af0 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/rag-eval.png and b/blog/2023-11-21-rag-evaluation/img/rag-eval.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/ragas-score.png b/blog/2023-11-21-rag-evaluation/img/ragas-score.png
index 328021923c..07bb3bbe98 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/ragas-score.png and b/blog/2023-11-21-rag-evaluation/img/ragas-score.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/sql-router.png b/blog/2023-11-21-rag-evaluation/img/sql-router.png
index ea89619a38..dc2b97734c 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/sql-router.png and b/blog/2023-11-21-rag-evaluation/img/sql-router.png differ
diff --git a/blog/2023-11-21-rag-evaluation/img/sub-question.png b/blog/2023-11-21-rag-evaluation/img/sub-question.png
index 862a2b541c..ecd3393f25 100644
Binary files a/blog/2023-11-21-rag-evaluation/img/sub-question.png and b/blog/2023-11-21-rag-evaluation/img/sub-question.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/any2any_D.png b/blog/2023-12-05-multimodal-RAG/img/any2any_D.png
index 2a6f7f9ba3..0d14c97783 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/any2any_D.png and b/blog/2023-12-05-multimodal-RAG/img/any2any_D.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/any2any_L.png b/blog/2023-12-05-multimodal-RAG/img/any2any_L.png
index 7ed3a6b0b3..16f169cbf0 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/any2any_L.png and b/blog/2023-12-05-multimodal-RAG/img/any2any_L.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/cross_modal.png b/blog/2023-12-05-multimodal-RAG/img/cross_modal.png
index cfb9c15b0c..8d896db2b2 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/cross_modal.png and b/blog/2023-12-05-multimodal-RAG/img/cross_modal.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/hero.png b/blog/2023-12-05-multimodal-RAG/img/hero.png
index 9d527b9bb3..05e50f8cfb 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/hero.png and b/blog/2023-12-05-multimodal-RAG/img/hero.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/image_a.png b/blog/2023-12-05-multimodal-RAG/img/image_a.png
index 0312dec553..6ba214c615 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/image_a.png and b/blog/2023-12-05-multimodal-RAG/img/image_a.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/image_b.png b/blog/2023-12-05-multimodal-RAG/img/image_b.png
index 7f9e4017b9..25d9900a35 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/image_b.png and b/blog/2023-12-05-multimodal-RAG/img/image_b.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/mmRAG_D.png b/blog/2023-12-05-multimodal-RAG/img/mmRAG_D.png
index 68d76801d6..40cc9fed71 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/mmRAG_D.png and b/blog/2023-12-05-multimodal-RAG/img/mmRAG_D.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/mm_RAG_L.png b/blog/2023-12-05-multimodal-RAG/img/mm_RAG_L.png
index a6e9c04eb4..98334f1082 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/mm_RAG_L.png and b/blog/2023-12-05-multimodal-RAG/img/mm_RAG_L.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/puppy.png b/blog/2023-12-05-multimodal-RAG/img/puppy.png
index 8205a46d0a..a260e97e42 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/puppy.png and b/blog/2023-12-05-multimodal-RAG/img/puppy.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/puppy_dalle.png b/blog/2023-12-05-multimodal-RAG/img/puppy_dalle.png
index e418dcdd55..68453a44e4 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/puppy_dalle.png and b/blog/2023-12-05-multimodal-RAG/img/puppy_dalle.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/unified_emb_D.png b/blog/2023-12-05-multimodal-RAG/img/unified_emb_D.png
index 8894fda3d2..9192814e82 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/unified_emb_D.png and b/blog/2023-12-05-multimodal-RAG/img/unified_emb_D.png differ
diff --git a/blog/2023-12-05-multimodal-RAG/img/unified_emb_L.png b/blog/2023-12-05-multimodal-RAG/img/unified_emb_L.png
index 8adeef32df..1da715c9d1 100644
Binary files a/blog/2023-12-05-multimodal-RAG/img/unified_emb_L.png and b/blog/2023-12-05-multimodal-RAG/img/unified_emb_L.png differ
diff --git a/blog/2023-12-19-weaviate-1-23-release/img/generative-anyscale.png b/blog/2023-12-19-weaviate-1-23-release/img/generative-anyscale.png
index 899780880d..b77cf62bb3 100644
Binary files a/blog/2023-12-19-weaviate-1-23-release/img/generative-anyscale.png and b/blog/2023-12-19-weaviate-1-23-release/img/generative-anyscale.png differ
diff --git a/blog/2023-12-26-weaviate-recap/img/Group_2596.png b/blog/2023-12-26-weaviate-recap/img/Group_2596.png
new file mode 100644
index 0000000000..e1fc3a3b5a
Binary files /dev/null and b/blog/2023-12-26-weaviate-recap/img/Group_2596.png differ
diff --git a/blog/2023-12-26-weaviate-recap/img/Group_2596_(1).png b/blog/2023-12-26-weaviate-recap/img/Group_2596_(1).png
deleted file mode 100644
index ea565b318a..0000000000
Binary files a/blog/2023-12-26-weaviate-recap/img/Group_2596_(1).png and /dev/null differ
diff --git a/blog/2023-12-26-weaviate-recap/img/Group_2597.png b/blog/2023-12-26-weaviate-recap/img/Group_2597.png
index ebe5b9887f..a750ed1122 100644
Binary files a/blog/2023-12-26-weaviate-recap/img/Group_2597.png and b/blog/2023-12-26-weaviate-recap/img/Group_2597.png differ
diff --git a/blog/2023-12-26-weaviate-recap/img/Recap_2023.png b/blog/2023-12-26-weaviate-recap/img/Recap_2023.png
index ead51206e9..b6b38642b0 100644
Binary files a/blog/2023-12-26-weaviate-recap/img/Recap_2023.png and b/blog/2023-12-26-weaviate-recap/img/Recap_2023.png differ
diff --git a/blog/2023-12-26-weaviate-recap/img/Weaviate-hero_(1).gif b/blog/2023-12-26-weaviate-recap/img/Weaviate-hero.gif
similarity index 100%
rename from blog/2023-12-26-weaviate-recap/img/Weaviate-hero_(1).gif
rename to blog/2023-12-26-weaviate-recap/img/Weaviate-hero.gif
diff --git a/blog/2023-12-26-weaviate-recap/img/hackathons.png b/blog/2023-12-26-weaviate-recap/img/hackathons.png
index 50520e5367..ce9fb78d6f 100644
Binary files a/blog/2023-12-26-weaviate-recap/img/hackathons.png and b/blog/2023-12-26-weaviate-recap/img/hackathons.png differ
diff --git a/blog/2023-12-26-weaviate-recap/img/hero.png b/blog/2023-12-26-weaviate-recap/img/hero.png
index 685f4713fa..7b612e3a60 100644
Binary files a/blog/2023-12-26-weaviate-recap/img/hero.png and b/blog/2023-12-26-weaviate-recap/img/hero.png differ
diff --git a/blog/2023-12-26-weaviate-recap/img/workshops.png b/blog/2023-12-26-weaviate-recap/img/workshops.png
new file mode 100644
index 0000000000..5e30e89ab8
Binary files /dev/null and b/blog/2023-12-26-weaviate-recap/img/workshops.png differ
diff --git a/blog/2023-12-26-weaviate-recap/img/workshops_3.png b/blog/2023-12-26-weaviate-recap/img/workshops_3.png
deleted file mode 100644
index 5375f64e82..0000000000
Binary files a/blog/2023-12-26-weaviate-recap/img/workshops_3.png and /dev/null differ
diff --git a/blog/2023-12-26-weaviate-recap/index.mdx b/blog/2023-12-26-weaviate-recap/index.mdx
index db8572e662..8bef4575b6 100644
--- a/blog/2023-12-26-weaviate-recap/index.mdx
+++ b/blog/2023-12-26-weaviate-recap/index.mdx
@@ -42,7 +42,7 @@ In the realm of Generative AI, especially with the emergence of OpenAI's ChatGPT
Our in-house projects, [Verba, an open-source RAG app](https://github.com/weaviate/Verba) and [Health Search](https://github.com/weaviate/healthsearch-demo) built by [Edward Schmuhl](https://www.linkedin.com/in/edwardschmuhl/), and other projects like [Leonie Monigatti's](https://www.linkedin.com/in/804250ab/) [Awesome-Moviate](https://github.com/weaviate-tutorials/awesome-moviate) and [Adam Chan](https://www.linkedin.com/in/itsajchan/)'s [Recommendation System](https://github.com/itsajchan/Create-JSON-Data-with-AI) have pushed the boundaries of what's possible with vector databases applicable to different industry use cases.
-.png)
+
### Community projects
@@ -73,7 +73,7 @@ Every week, [Zain](https://www.linkedin.com/in/zainhas/), [JP](https://www.linke
Later in the year, we teamed up with [DeepLearningAI](https://www.deeplearning.ai/) to create a short course with [Sebastian](https://www.linkedin.com/in/sebawita/) and [Zain](https://www.linkedin.com/in/zainhas/), "[Vector Databases: from Embeddings to Applications with Weaviate](https://www.deeplearning.ai/short-courses/vector-databases-embeddings-applications/)." It’s been a year packed with learning.
-
+
## Building a Global Community
@@ -100,7 +100,7 @@ Nominate your hero [here](https://docs.google.com/forms/d/e/1FAIpQLSePtgvTPZGx2g
As we keep building, learning, and collaborating, we're excited to see the innovations that 2024 will bring. 💫
-.gif)
+
import WhatNext from '/_includes/what-next.mdx'
diff --git a/blog/2024-01-23-typescript-multimodal-search/img/hero.png b/blog/2024-01-23-typescript-multimodal-search/img/hero.png
index d3bbcfd786..eab5d2d6a9 100644
Binary files a/blog/2024-01-23-typescript-multimodal-search/img/hero.png and b/blog/2024-01-23-typescript-multimodal-search/img/hero.png differ
diff --git a/blog/2024-01-23-typescript-multimodal-search/img/multimodal-1.png b/blog/2024-01-23-typescript-multimodal-search/img/multimodal-1.png
index bc7466e44a..d01405217b 100644
Binary files a/blog/2024-01-23-typescript-multimodal-search/img/multimodal-1.png and b/blog/2024-01-23-typescript-multimodal-search/img/multimodal-1.png differ
diff --git a/blog/2024-01-23-typescript-multimodal-search/img/multimodal-2.png b/blog/2024-01-23-typescript-multimodal-search/img/multimodal-2.png
index d636326114..dbfd351248 100644
Binary files a/blog/2024-01-23-typescript-multimodal-search/img/multimodal-2.png and b/blog/2024-01-23-typescript-multimodal-search/img/multimodal-2.png differ
diff --git a/blog/2024-01-23-typescript-multimodal-search/img/mumose-1.gif b/blog/2024-01-23-typescript-multimodal-search/img/mumose-1.gif
deleted file mode 100644
index 6d5cb56386..0000000000
Binary files a/blog/2024-01-23-typescript-multimodal-search/img/mumose-1.gif and /dev/null differ
diff --git a/blog/2024-01-23-typescript-multimodal-search/img/project-demo.mp4 b/blog/2024-01-23-typescript-multimodal-search/img/project-demo.mp4
new file mode 100644
index 0000000000..4ffebf20b3
Binary files /dev/null and b/blog/2024-01-23-typescript-multimodal-search/img/project-demo.mp4 differ
diff --git a/blog/2024-01-23-typescript-multimodal-search/index.mdx b/blog/2024-01-23-typescript-multimodal-search/index.mdx
index e64d79e708..0a7c9e7856 100644
--- a/blog/2024-01-23-typescript-multimodal-search/index.mdx
+++ b/blog/2024-01-23-typescript-multimodal-search/index.mdx
@@ -514,7 +514,12 @@ Here was call the `vectorSearch()` server action we created above and pass the q
### Final Result
Then you should be able to search like this
-
+import multimodalVideo from './img/project-demo.mp4';
+
+
## Conclusion
diff --git a/blog/2024-02-01-py-client-v4-release/index.mdx b/blog/2024-02-01-py-client-v4-release/index.mdx
index 37d58e8076..54b8717c9d 100644
--- a/blog/2024-02-01-py-client-v4-release/index.mdx
+++ b/blog/2024-02-01-py-client-v4-release/index.mdx
@@ -12,6 +12,7 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PythonCode from '!!raw-loader!/_includes/code/client-libraries/python_v4.py';
+import PythonV3Code from '!!raw-loader!/_includes/code/connections/connect-python-v3.py';
import ManageCollectionsCode from '!!raw-loader!/_includes/code/howto/manage-data.collections.py';
@@ -106,7 +107,7 @@ Here is an example that really captures the difference between the `v4` and `v3`
text={CollectionsV3}
startMarker="# START CreateCollectionWithProperties"
endMarker="# END CreateCollectionWithProperties"
- language="python"
+ language="pyv3"
/>
@@ -180,7 +181,7 @@ Direct instantiation with full control over the parameters is also possible, too
\ No newline at end of file
+
diff --git a/blog/2024-03-13-weaviate-champions/img/framework.png b/blog/2024-03-13-weaviate-champions/img/framework.png
index 9aaa9ef765..91ff90cc80 100644
Binary files a/blog/2024-03-13-weaviate-champions/img/framework.png and b/blog/2024-03-13-weaviate-champions/img/framework.png differ
diff --git a/blog/2024-03-13-weaviate-champions/img/hero.png b/blog/2024-03-13-weaviate-champions/img/hero.png
index 6a2b9b56b5..b094df541c 100644
Binary files a/blog/2024-03-13-weaviate-champions/img/hero.png and b/blog/2024-03-13-weaviate-champions/img/hero.png differ
diff --git a/blog/2024-03-13-weaviate-champions/img/map..001.png b/blog/2024-03-13-weaviate-champions/img/map..001.png
deleted file mode 100644
index ddc819f549..0000000000
Binary files a/blog/2024-03-13-weaviate-champions/img/map..001.png and /dev/null differ
diff --git a/blog/2024-03-13-weaviate-champions/img/patrice_bourgougnon.png b/blog/2024-03-13-weaviate-champions/img/patrice_bourgougnon.png
index 513d93d0c7..61c15b77ee 100644
Binary files a/blog/2024-03-13-weaviate-champions/img/patrice_bourgougnon.png and b/blog/2024-03-13-weaviate-champions/img/patrice_bourgougnon.png differ
diff --git a/blog/2024-03-13-weaviate-champions/img/ron_parker.png b/blog/2024-03-13-weaviate-champions/img/ron_parker.png
index fb1d5c49a8..08aee34fc2 100644
Binary files a/blog/2024-03-13-weaviate-champions/img/ron_parker.png and b/blog/2024-03-13-weaviate-champions/img/ron_parker.png differ
diff --git a/blog/2024-03-13-weaviate-champions/img/travis_rehl.png b/blog/2024-03-13-weaviate-champions/img/travis_rehl.png
index 5a63c25eab..abd88eec32 100644
Binary files a/blog/2024-03-13-weaviate-champions/img/travis_rehl.png and b/blog/2024-03-13-weaviate-champions/img/travis_rehl.png differ
diff --git a/blog/2024-03-19-hurricane-blog-gfls/img/ALTO.png b/blog/2024-03-19-hurricane-blog-gfls/img/ALTO.png
index bcb1a27c05..6e5012fb95 100644
Binary files a/blog/2024-03-19-hurricane-blog-gfls/img/ALTO.png and b/blog/2024-03-19-hurricane-blog-gfls/img/ALTO.png differ
diff --git a/blog/2024-03-19-hurricane-blog-gfls/img/hero.png b/blog/2024-03-19-hurricane-blog-gfls/img/hero.png
index 1bbbeabe53..e0e8cf92fe 100644
Binary files a/blog/2024-03-19-hurricane-blog-gfls/img/hero.png and b/blog/2024-03-19-hurricane-blog-gfls/img/hero.png differ
diff --git a/blog/2024-03-20-typescript-client-beta/img/inverted-index-hints.png b/blog/2024-03-20-typescript-client-beta/img/inverted-index-hints.png
index f07f9404fc..3327d27f03 100644
Binary files a/blog/2024-03-20-typescript-client-beta/img/inverted-index-hints.png and b/blog/2024-03-20-typescript-client-beta/img/inverted-index-hints.png differ
diff --git a/blog/2024-03-28-hybrid-search-typescript/img/quote-finder-screenshot.png b/blog/2024-03-28-hybrid-search-typescript/img/quote-finder-screenshot.png
index 5e3d129860..f1a2f512d1 100644
Binary files a/blog/2024-03-28-hybrid-search-typescript/img/quote-finder-screenshot.png and b/blog/2024-03-28-hybrid-search-typescript/img/quote-finder-screenshot.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/bq_vecs.png b/blog/2024-04-02-binary-quantization/img/bq_vecs.png
index 39dda44750..c70956943a 100644
Binary files a/blog/2024-04-02-binary-quantization/img/bq_vecs.png and b/blog/2024-04-02-binary-quantization/img/bq_vecs.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/hero.png b/blog/2024-04-02-binary-quantization/img/hero.png
index def1f6d1dd..d4c3c98d8a 100644
Binary files a/blog/2024-04-02-binary-quantization/img/hero.png and b/blog/2024-04-02-binary-quantization/img/hero.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image11.png b/blog/2024-04-02-binary-quantization/img/image11.png
index 8eb6015d8f..de384aa6ac 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image11.png and b/blog/2024-04-02-binary-quantization/img/image11.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image13.png b/blog/2024-04-02-binary-quantization/img/image13.png
index 7217185db2..cc3c12f16c 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image13.png and b/blog/2024-04-02-binary-quantization/img/image13.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image14.jpg b/blog/2024-04-02-binary-quantization/img/image14.jpg
index 6d2126f855..c9b81b97d8 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image14.jpg and b/blog/2024-04-02-binary-quantization/img/image14.jpg differ
diff --git a/blog/2024-04-02-binary-quantization/img/image14.png b/blog/2024-04-02-binary-quantization/img/image14.png
new file mode 100644
index 0000000000..30296ca455
Binary files /dev/null and b/blog/2024-04-02-binary-quantization/img/image14.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image15.png b/blog/2024-04-02-binary-quantization/img/image15.png
index 0802381cfa..68606a3262 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image15.png and b/blog/2024-04-02-binary-quantization/img/image15.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image16.png b/blog/2024-04-02-binary-quantization/img/image16.png
index 9088437873..90ce2e1c35 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image16.png and b/blog/2024-04-02-binary-quantization/img/image16.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image17.png b/blog/2024-04-02-binary-quantization/img/image17.png
index 1b6831e7a8..813a50ce8f 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image17.png and b/blog/2024-04-02-binary-quantization/img/image17.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image18.png b/blog/2024-04-02-binary-quantization/img/image18.png
index 60297f8b42..7d98413cc7 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image18.png and b/blog/2024-04-02-binary-quantization/img/image18.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image19.png b/blog/2024-04-02-binary-quantization/img/image19.png
index 6305b74be5..328c5343a8 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image19.png and b/blog/2024-04-02-binary-quantization/img/image19.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image20.png b/blog/2024-04-02-binary-quantization/img/image20.png
index 555da380bc..d0470e99a0 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image20.png and b/blog/2024-04-02-binary-quantization/img/image20.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image21.png b/blog/2024-04-02-binary-quantization/img/image21.png
index 13f80debf0..3d9f5dbfd4 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image21.png and b/blog/2024-04-02-binary-quantization/img/image21.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image22.png b/blog/2024-04-02-binary-quantization/img/image22.png
index 9d6fed61e8..f0219ff416 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image22.png and b/blog/2024-04-02-binary-quantization/img/image22.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image24.png b/blog/2024-04-02-binary-quantization/img/image24.png
index 971b4fee88..a9c582c161 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image24.png and b/blog/2024-04-02-binary-quantization/img/image24.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image3.png b/blog/2024-04-02-binary-quantization/img/image3.png
index 5e83ee8542..926206276d 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image3.png and b/blog/2024-04-02-binary-quantization/img/image3.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image4.png b/blog/2024-04-02-binary-quantization/img/image4.png
index d614129c40..baf72db3ef 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image4.png and b/blog/2024-04-02-binary-quantization/img/image4.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image6.png b/blog/2024-04-02-binary-quantization/img/image6.png
index c8320a68d2..16749069f7 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image6.png and b/blog/2024-04-02-binary-quantization/img/image6.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image7.png b/blog/2024-04-02-binary-quantization/img/image7.png
index 58040dc83f..45aa66df1a 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image7.png and b/blog/2024-04-02-binary-quantization/img/image7.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image8.png b/blog/2024-04-02-binary-quantization/img/image8.png
index 24870d8265..c2d14f4788 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image8.png and b/blog/2024-04-02-binary-quantization/img/image8.png differ
diff --git a/blog/2024-04-02-binary-quantization/img/image9.png b/blog/2024-04-02-binary-quantization/img/image9.png
index 5b021932b1..8148ceb383 100644
Binary files a/blog/2024-04-02-binary-quantization/img/image9.png and b/blog/2024-04-02-binary-quantization/img/image9.png differ
diff --git a/blog/2024-04-17-dspy-optimizers/img/RAG-optimization-dashboard.png b/blog/2024-04-17-dspy-optimizers/img/RAG-optimization-dashboard.png
index a8de3b82a7..a25b182e1a 100644
Binary files a/blog/2024-04-17-dspy-optimizers/img/RAG-optimization-dashboard.png and b/blog/2024-04-17-dspy-optimizers/img/RAG-optimization-dashboard.png differ
diff --git a/blog/2024-04-17-dspy-optimizers/img/hero.png b/blog/2024-04-17-dspy-optimizers/img/hero.png
index 9ddc165da3..30522c4948 100644
Binary files a/blog/2024-04-17-dspy-optimizers/img/hero.png and b/blog/2024-04-17-dspy-optimizers/img/hero.png differ
diff --git a/blog/2024-04-23-unbody-weaviate/img/diagram-1.png b/blog/2024-04-23-unbody-weaviate/img/diagram-1.png
index 65d79d29f7..c29aa8151a 100644
Binary files a/blog/2024-04-23-unbody-weaviate/img/diagram-1.png and b/blog/2024-04-23-unbody-weaviate/img/diagram-1.png differ
diff --git a/blog/2024-04-23-unbody-weaviate/img/diagram-2.png b/blog/2024-04-23-unbody-weaviate/img/diagram-2.png
index 45a564357d..ca6c224fa1 100644
Binary files a/blog/2024-04-23-unbody-weaviate/img/diagram-2.png and b/blog/2024-04-23-unbody-weaviate/img/diagram-2.png differ
diff --git a/blog/2024-04-23-unbody-weaviate/img/diagram-3.png b/blog/2024-04-23-unbody-weaviate/img/diagram-3.png
index 3fcecb1b3d..e279dff3ec 100644
Binary files a/blog/2024-04-23-unbody-weaviate/img/diagram-3.png and b/blog/2024-04-23-unbody-weaviate/img/diagram-3.png differ
diff --git a/blog/2024-04-23-unbody-weaviate/img/hero.png b/blog/2024-04-23-unbody-weaviate/img/hero.png
index 53f58f731c..ebabe7b353 100644
Binary files a/blog/2024-04-23-unbody-weaviate/img/hero.png and b/blog/2024-04-23-unbody-weaviate/img/hero.png differ
diff --git a/blog/2024-04-23-unbody-weaviate/img/search-plugin.png b/blog/2024-04-23-unbody-weaviate/img/search-plugin.png
index adeb2fc5ae..bcec5f5a8b 100644
Binary files a/blog/2024-04-23-unbody-weaviate/img/search-plugin.png and b/blog/2024-04-23-unbody-weaviate/img/search-plugin.png differ
diff --git a/blog/2024-04-24-next-recap/img/group-pic.jpg b/blog/2024-04-24-next-recap/img/group-pic.jpg
new file mode 100644
index 0000000000..c320ec51eb
Binary files /dev/null and b/blog/2024-04-24-next-recap/img/group-pic.jpg differ
diff --git a/blog/2024-04-24-next-recap/img/group-pic.png b/blog/2024-04-24-next-recap/img/group-pic.png
deleted file mode 100644
index 5735f4a0ab..0000000000
Binary files a/blog/2024-04-24-next-recap/img/group-pic.png and /dev/null differ
diff --git a/blog/2024-04-24-next-recap/index.mdx b/blog/2024-04-24-next-recap/index.mdx
index 9f4fe6f91b..866d616f5e 100644
--- a/blog/2024-04-24-next-recap/index.mdx
+++ b/blog/2024-04-24-next-recap/index.mdx
@@ -53,7 +53,7 @@ Next is Google Cloud’s big annual conference, so of course we were waiting to
## Weaviate winners
We’re a fully remote company, so our team has a ton of fun when we’re together in person. It’s always a good time getting the crowd excited for our booth giveaways and seeing the joy in their faces when they win (or don’t and can have a good laugh about it). This time around we gave away clicky-clacky Keychron keyboards and some crowd-favorite Lego sets.
-
+
## What comes next?
**Google I/O**: Google’s developer conference is happening on May 14th in Mountain View, CA. Keep an eye out for some big announcements, maybe even from your favorite vector database. 😉
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/img/hero.jpg b/blog/2024-04-30-find-waldo-with-weaviate/img/hero.jpg
new file mode 100644
index 0000000000..7a912cb598
Binary files /dev/null and b/blog/2024-04-30-find-waldo-with-weaviate/img/hero.jpg differ
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/img/hero.png b/blog/2024-04-30-find-waldo-with-weaviate/img/hero.png
deleted file mode 100644
index 518371b315..0000000000
Binary files a/blog/2024-04-30-find-waldo-with-weaviate/img/hero.png and /dev/null differ
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/img/image1.png b/blog/2024-04-30-find-waldo-with-weaviate/img/image1.png
index 895bf9772f..caf31e296b 100644
Binary files a/blog/2024-04-30-find-waldo-with-weaviate/img/image1.png and b/blog/2024-04-30-find-waldo-with-weaviate/img/image1.png differ
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/img/image2.jpg b/blog/2024-04-30-find-waldo-with-weaviate/img/image2.jpg
new file mode 100644
index 0000000000..b8e048cb1c
Binary files /dev/null and b/blog/2024-04-30-find-waldo-with-weaviate/img/image2.jpg differ
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/img/image2.png b/blog/2024-04-30-find-waldo-with-weaviate/img/image2.png
deleted file mode 100644
index ac58a5a21d..0000000000
Binary files a/blog/2024-04-30-find-waldo-with-weaviate/img/image2.png and /dev/null differ
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/img/image3.png b/blog/2024-04-30-find-waldo-with-weaviate/img/image3.png
index ef2e8717a0..ab93fbf802 100644
Binary files a/blog/2024-04-30-find-waldo-with-weaviate/img/image3.png and b/blog/2024-04-30-find-waldo-with-weaviate/img/image3.png differ
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/img/image4.jpg b/blog/2024-04-30-find-waldo-with-weaviate/img/image4.jpg
index 52b65bdf20..190605f6fa 100644
Binary files a/blog/2024-04-30-find-waldo-with-weaviate/img/image4.jpg and b/blog/2024-04-30-find-waldo-with-weaviate/img/image4.jpg differ
diff --git a/blog/2024-04-30-find-waldo-with-weaviate/index.mdx b/blog/2024-04-30-find-waldo-with-weaviate/index.mdx
index 67e9bae188..f7cb1da563 100644
--- a/blog/2024-04-30-find-waldo-with-weaviate/index.mdx
+++ b/blog/2024-04-30-find-waldo-with-weaviate/index.mdx
@@ -5,10 +5,10 @@ description: 'Dive into using Weaviate for image recognition to find the "needle
authors: [jon]
date: 2024-04-30
tags: ['how-to', 'search']
-image: ./img/hero.png
+image: ./img/hero.jpg
---
-
+
:::info GitHub Repo
@@ -140,7 +140,7 @@ else:
reassembled_image.show() # This will display the reassembled image with highlighted parts
```
-
+
It worked! Using Weaviate I was able to find Waldo in a group of images. This has a lot of implications on the application. Using the same technique you could utilize a large corpus of image data to create an image lookup system.
diff --git a/blog/2024-05-08-Modal-and-Weaviate/img/example.png b/blog/2024-05-08-Modal-and-Weaviate/img/example.png
index ca41cf1b63..364dc52627 100644
Binary files a/blog/2024-05-08-Modal-and-Weaviate/img/example.png and b/blog/2024-05-08-Modal-and-Weaviate/img/example.png differ
diff --git a/blog/2024-05-08-Modal-and-Weaviate/img/hero.png b/blog/2024-05-08-Modal-and-Weaviate/img/hero.png
index 9414fabffd..904ba1eb58 100644
Binary files a/blog/2024-05-08-Modal-and-Weaviate/img/hero.png and b/blog/2024-05-08-Modal-and-Weaviate/img/hero.png differ
diff --git a/blog/2024-05-08-Modal-and-Weaviate/index.mdx b/blog/2024-05-08-Modal-and-Weaviate/index.mdx
index 01485eaacb..ce49576e37 100644
--- a/blog/2024-05-08-Modal-and-Weaviate/index.mdx
+++ b/blog/2024-05-08-Modal-and-Weaviate/index.mdx
@@ -61,9 +61,9 @@ In this section, we walk through some of the key Weaviate features we used to im
5. [Batch Imports](#batch-imports)
### Async Indexing
-Importing data into a vector database is a two-step process: 1. Object creation, and 2. Vector index construction. Typically, the database won’t receive another batch of objects until it has finished indexing the prior batch. Weaviate `1.22` introduced [async indexing](/developers/weaviate/release-notes/release_1_22#asynchronous-vector-indexing-experimental) to speed up data ingestion.
+Importing data into a vector database is a two-step process: 1. Object creation, and 2. Vector index construction. Typically, the database won’t receive another batch of objects until it has finished indexing the prior batch. Weaviate `1.22` introduced [async indexing](/developers/weaviate/release-notes/older-releases/release_1_22#asynchronous-vector-indexing-experimental) to speed up data ingestion.
-Async indexing speeds up object ingestion by separating it from the vector index construction process. Rather than waiting for the new objects to be indexed, the objects are put in an in-memory queue before getting indexed. Since we imported millions of objects, it was important to enable this feature to reduce the import time.
+Async indexing speeds up object ingestion by separating it from the [vector index](/developers/weaviate/concepts/vector-index) construction process. Rather than waiting for the new objects to be indexed, the objects are put in an in-memory queue before getting indexed. Since we imported millions of objects, it was important to enable this feature to reduce the import time.
For open-source users, you can enable it by adding `ASYNC_INDEXING=true` in your environment variable, or you can enable it through Weaviate Cloud.
diff --git a/blog/2024-05-14-instabase-and-weaviate/img/hero.png b/blog/2024-05-14-instabase-and-weaviate/img/hero.png
index 6859e59fea..bf603c072b 100644
Binary files a/blog/2024-05-14-instabase-and-weaviate/img/hero.png and b/blog/2024-05-14-instabase-and-weaviate/img/hero.png differ
diff --git a/blog/2024-05-20-weaviate-1-25-release/_core-1-25-include.mdx b/blog/2024-05-20-weaviate-1-25-release/_core-1-25-include.mdx
index 224ae6291b..fa60b02253 100644
--- a/blog/2024-05-20-weaviate-1-25-release/_core-1-25-include.mdx
+++ b/blog/2024-05-20-weaviate-1-25-release/_core-1-25-include.mdx
@@ -66,6 +66,10 @@ This release brings important new integrations with third party APIs.
### OctoAI integration
+import OctoAIDeprecationNote from '/developers/weaviate/model-providers/octoai/_includes/octoai_deprecation.md';
+
+
+
With version 1.25 we’re announcing an integration with OctoAI which will make it even easier for users to access many open source embedding and language models such as Llama3-70b, Mixtral-8x22b and more.
We are releasing two integrations: text2vec-octoai and generative-octoai that integrate Weaviate and the OctoAI service. OctoAI provides hosted inference services for embedding models and large language models.
diff --git a/blog/2024-05-20-weaviate-1-25-release/img/dynamic.png b/blog/2024-05-20-weaviate-1-25-release/img/dynamic.png
index d611ff5133..04d71c822e 100644
Binary files a/blog/2024-05-20-weaviate-1-25-release/img/dynamic.png and b/blog/2024-05-20-weaviate-1-25-release/img/dynamic.png differ
diff --git a/blog/2024-05-20-weaviate-1-25-release/img/hero.png b/blog/2024-05-20-weaviate-1-25-release/img/hero.png
index 5ee8c74967..2729e31ee8 100644
Binary files a/blog/2024-05-20-weaviate-1-25-release/img/hero.png and b/blog/2024-05-20-weaviate-1-25-release/img/hero.png differ
diff --git a/blog/2024-05-20-weaviate-1-25-release/img/raft-weaviate.png b/blog/2024-05-20-weaviate-1-25-release/img/raft-weaviate.png
index 558029e857..34926b5b72 100644
Binary files a/blog/2024-05-20-weaviate-1-25-release/img/raft-weaviate.png and b/blog/2024-05-20-weaviate-1-25-release/img/raft-weaviate.png differ
diff --git a/blog/2024-05-21-local-rag-ollama/img/hero.png b/blog/2024-05-21-local-rag-ollama/img/hero.png
index c64abb6974..a8cc3561a8 100644
Binary files a/blog/2024-05-21-local-rag-ollama/img/hero.png and b/blog/2024-05-21-local-rag-ollama/img/hero.png differ
diff --git a/blog/2024-05-21-local-rag-ollama/img/rag-ollama-diagram.png b/blog/2024-05-21-local-rag-ollama/img/rag-ollama-diagram.png
index dc1668a4d9..cd8f82e6ff 100644
Binary files a/blog/2024-05-21-local-rag-ollama/img/rag-ollama-diagram.png and b/blog/2024-05-21-local-rag-ollama/img/rag-ollama-diagram.png differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/example.jpeg b/blog/2024-05-28-retrieval-evaluation-metrics/img/example.jpeg
deleted file mode 100644
index ad23eb5947..0000000000
Binary files a/blog/2024-05-28-retrieval-evaluation-metrics/img/example.jpeg and /dev/null differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/example.jpg b/blog/2024-05-28-retrieval-evaluation-metrics/img/example.jpg
new file mode 100644
index 0000000000..189d1a7706
Binary files /dev/null and b/blog/2024-05-28-retrieval-evaluation-metrics/img/example.jpg differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/hero.png b/blog/2024-05-28-retrieval-evaluation-metrics/img/hero.png
index 526412f73a..27d513c51d 100644
Binary files a/blog/2024-05-28-retrieval-evaluation-metrics/img/hero.png and b/blog/2024-05-28-retrieval-evaluation-metrics/img/hero.png differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/map.jpeg b/blog/2024-05-28-retrieval-evaluation-metrics/img/map.jpeg
deleted file mode 100644
index 8b450b4239..0000000000
Binary files a/blog/2024-05-28-retrieval-evaluation-metrics/img/map.jpeg and /dev/null differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/map.jpg b/blog/2024-05-28-retrieval-evaluation-metrics/img/map.jpg
new file mode 100644
index 0000000000..4111628598
Binary files /dev/null and b/blog/2024-05-28-retrieval-evaluation-metrics/img/map.jpg differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/mrr.jpeg b/blog/2024-05-28-retrieval-evaluation-metrics/img/mrr.jpeg
deleted file mode 100644
index 8b8ece0eb1..0000000000
Binary files a/blog/2024-05-28-retrieval-evaluation-metrics/img/mrr.jpeg and /dev/null differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/mrr.jpg b/blog/2024-05-28-retrieval-evaluation-metrics/img/mrr.jpg
new file mode 100644
index 0000000000..b30681308e
Binary files /dev/null and b/blog/2024-05-28-retrieval-evaluation-metrics/img/mrr.jpg differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/ndcg.jpeg b/blog/2024-05-28-retrieval-evaluation-metrics/img/ndcg.jpeg
deleted file mode 100644
index 4f02b522ac..0000000000
Binary files a/blog/2024-05-28-retrieval-evaluation-metrics/img/ndcg.jpeg and /dev/null differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/ndcg.jpg b/blog/2024-05-28-retrieval-evaluation-metrics/img/ndcg.jpg
new file mode 100644
index 0000000000..fd1eaf74aa
Binary files /dev/null and b/blog/2024-05-28-retrieval-evaluation-metrics/img/ndcg.jpg differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/precision.jpeg b/blog/2024-05-28-retrieval-evaluation-metrics/img/precision.jpeg
deleted file mode 100644
index e934b70422..0000000000
Binary files a/blog/2024-05-28-retrieval-evaluation-metrics/img/precision.jpeg and /dev/null differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/precision.jpg b/blog/2024-05-28-retrieval-evaluation-metrics/img/precision.jpg
new file mode 100644
index 0000000000..14a6b5ae7f
Binary files /dev/null and b/blog/2024-05-28-retrieval-evaluation-metrics/img/precision.jpg differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/recall.jpeg b/blog/2024-05-28-retrieval-evaluation-metrics/img/recall.jpeg
deleted file mode 100644
index 67349f435f..0000000000
Binary files a/blog/2024-05-28-retrieval-evaluation-metrics/img/recall.jpeg and /dev/null differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/img/recall.jpg b/blog/2024-05-28-retrieval-evaluation-metrics/img/recall.jpg
new file mode 100644
index 0000000000..5bf8373617
Binary files /dev/null and b/blog/2024-05-28-retrieval-evaluation-metrics/img/recall.jpg differ
diff --git a/blog/2024-05-28-retrieval-evaluation-metrics/index.mdx b/blog/2024-05-28-retrieval-evaluation-metrics/index.mdx
index e066c3a832..c17e0e6000 100644
--- a/blog/2024-05-28-retrieval-evaluation-metrics/index.mdx
+++ b/blog/2024-05-28-retrieval-evaluation-metrics/index.mdx
@@ -18,7 +18,7 @@ This article gives a comprehensive overview of commonly used evaluation metrics
To explain how each of the following metrics is calculated, this article uses a minimal example dataset of eight pastries - to stay in line with the bagel example our colleagues used in their recent [discussion on how to choose and evaluate an embedding model](https://youtu.be/lpdN3aw-yTg?si=CHzV9zS_L2-rdUel) - as shown below.
-
+
We will use the [`pytrec_eval` Python package](https://github.com/cvangysel/pytrec_eval) to showcase how you can evaluate your retrieval or recommendation pipelines in your code. Check out the code in the [related GitHub repository](https://github.com/weaviate/recipes/blob/main/weaviate-features/evaluation/evaluation_metrics_information_retrieval.ipynb).
@@ -50,7 +50,7 @@ $$
=\frac{\text{Number of relevant items in K}}{\text{Total number of items in K}}
$$
-
+
With `pytrec_eval` you can evaluate your search results as follows:
@@ -95,7 +95,7 @@ $$
=\frac{\text{Number of relevant items in K}}{\text{Total number of relevant items}}
$$
-
+
With `pytrec_eval` you can evaluate your search results as follows:
@@ -126,7 +126,7 @@ $$
\text{RR} = \frac{1}{\text{Rank of the first relevant item}}
$$
-
+
Note that the MRR metric is not available in the `pytrec_eval` package as of this writing.
@@ -144,7 +144,7 @@ $$
\text{AP@K} = \frac{1}{N}\sum_{k=1}^{K}\text{Precision(k)}\times \text{rel(k)}
$$
-
+
This is also the default metric used in the [Massive Text Embedding Benchmark (MTEB) Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) for the **Reranking** category.
@@ -197,7 +197,7 @@ $$
\text{DCG@K} = \sum_{k=1}^K\frac{\text{rel}_i}{\log_2(i+1)}
$$
-
+
This is the default metric used in the [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) for the **Retrieval** category.
diff --git a/blog/2024-06-04-how-to-choose-an-embedding-model/img/embedding_model.png b/blog/2024-06-04-how-to-choose-an-embedding-model/img/embedding_model.png
index 4930ad4550..38fd5f7474 100644
Binary files a/blog/2024-06-04-how-to-choose-an-embedding-model/img/embedding_model.png and b/blog/2024-06-04-how-to-choose-an-embedding-model/img/embedding_model.png differ
diff --git a/blog/2024-06-04-how-to-choose-an-embedding-model/img/hero.png b/blog/2024-06-04-how-to-choose-an-embedding-model/img/hero.png
index 40ce8340d7..39f55ff94e 100644
Binary files a/blog/2024-06-04-how-to-choose-an-embedding-model/img/hero.png and b/blog/2024-06-04-how-to-choose-an-embedding-model/img/hero.png differ
diff --git a/blog/2024-06-04-how-to-choose-an-embedding-model/img/mteb.png b/blog/2024-06-04-how-to-choose-an-embedding-model/img/mteb.png
index cbea8a00dd..ef911e32f4 100644
Binary files a/blog/2024-06-04-how-to-choose-an-embedding-model/img/mteb.png and b/blog/2024-06-04-how-to-choose-an-embedding-model/img/mteb.png differ
diff --git a/blog/2024-06-18-openais-matryoshka-embeddings/img/MRLvar.png b/blog/2024-06-18-openais-matryoshka-embeddings/img/MRLvar.png
index 2ad276bf48..204e9f52fc 100644
Binary files a/blog/2024-06-18-openais-matryoshka-embeddings/img/MRLvar.png and b/blog/2024-06-18-openais-matryoshka-embeddings/img/MRLvar.png differ
diff --git a/blog/2024-06-18-openais-matryoshka-embeddings/img/Matryoshka_formula.png b/blog/2024-06-18-openais-matryoshka-embeddings/img/Matryoshka_formula.png
new file mode 100644
index 0000000000..c1299e7e94
Binary files /dev/null and b/blog/2024-06-18-openais-matryoshka-embeddings/img/Matryoshka_formula.png differ
diff --git a/blog/2024-06-18-openais-matryoshka-embeddings/img/Matryoshka_representation_learning.png b/blog/2024-06-18-openais-matryoshka-embeddings/img/Matryoshka_representation_learning.png
index f325d1be21..f5c3306af1 100644
Binary files a/blog/2024-06-18-openais-matryoshka-embeddings/img/Matryoshka_representation_learning.png and b/blog/2024-06-18-openais-matryoshka-embeddings/img/Matryoshka_representation_learning.png differ
diff --git a/blog/2024-06-18-openais-matryoshka-embeddings/img/OLDvar.jpeg b/blog/2024-06-18-openais-matryoshka-embeddings/img/OLDvar.jpeg
deleted file mode 100644
index 42791a0666..0000000000
Binary files a/blog/2024-06-18-openais-matryoshka-embeddings/img/OLDvar.jpeg and /dev/null differ
diff --git a/blog/2024-06-18-openais-matryoshka-embeddings/img/OLDvar.png b/blog/2024-06-18-openais-matryoshka-embeddings/img/OLDvar.png
new file mode 100644
index 0000000000..66ded07820
Binary files /dev/null and b/blog/2024-06-18-openais-matryoshka-embeddings/img/OLDvar.png differ
diff --git a/blog/2024-06-18-openais-matryoshka-embeddings/index.mdx b/blog/2024-06-18-openais-matryoshka-embeddings/index.mdx
index 7222703494..c1c45d5263 100644
--- a/blog/2024-06-18-openais-matryoshka-embeddings/index.mdx
+++ b/blog/2024-06-18-openais-matryoshka-embeddings/index.mdx
@@ -34,6 +34,8 @@ Retrieval performance over representation size for Matryoshka embeddings. (This
The magic behind how MRL is carried out is all in the loss function which is optimized when training these models! If previously the loss function was **L**, for MRL we break down the loss function into the sum of the losses on individual vector dimension ranges: `Loss_Total = L(upto 8d) + L(upto 16d) + L(upto 32d) + ... + L(upto 2048d)`. With this new nested loss function there is an incentive for the model to capture information in each sub-section of the vector.
+
+
After modifying the loss you get these truncatable vectors for free wihtout any additional costs - this works on almost all loss functions and pre-existing models can be finetuned to output MRL vectors! This means that MRL is very easy to adopt and apply to pretrained models.
## Detailed Analysis of Matryoshka Embeddings
@@ -45,7 +47,7 @@ In this analysis below we sampled 10k random vectors from the dataset above and
|Matryoshka Vectors | Non-Matryoshka Vectors |
|:-------------------------:|:-------------------------: |
-|  |  |
+|  |  |
This graphs shows the smoothed standard deviation per dimension across 10k random samples from DBpedia embedded with the new `text-emb3-large` model and the old `ada-002` model.
diff --git a/blog/2024-06-25-ny-roadshow-recap/img/hero.png b/blog/2024-06-25-ny-roadshow-recap/img/hero.png
index 324db6f705..29bafee611 100644
Binary files a/blog/2024-06-25-ny-roadshow-recap/img/hero.png and b/blog/2024-06-25-ny-roadshow-recap/img/hero.png differ
diff --git a/blog/2024-07-09-verba-with-local-llm/img/cluster-api-key.png b/blog/2024-07-09-verba-with-local-llm/img/cluster-api-key.png
index 47d90d454d..48e37b55c6 100644
Binary files a/blog/2024-07-09-verba-with-local-llm/img/cluster-api-key.png and b/blog/2024-07-09-verba-with-local-llm/img/cluster-api-key.png differ
diff --git a/blog/2024-07-09-verba-with-local-llm/img/cluster-details.png b/blog/2024-07-09-verba-with-local-llm/img/cluster-details.png
index 92d5ed2e71..d8af24566c 100644
Binary files a/blog/2024-07-09-verba-with-local-llm/img/cluster-details.png and b/blog/2024-07-09-verba-with-local-llm/img/cluster-details.png differ
diff --git a/blog/2024-07-09-verba-with-local-llm/img/hero.png b/blog/2024-07-09-verba-with-local-llm/img/hero.png
index f6bbacb93f..b2ed472d3a 100644
Binary files a/blog/2024-07-09-verba-with-local-llm/img/hero.png and b/blog/2024-07-09-verba-with-local-llm/img/hero.png differ
diff --git a/blog/2024-07-09-verba-with-local-llm/img/verba-browser.png b/blog/2024-07-09-verba-with-local-llm/img/verba-browser.png
index 934956d47c..6f9ccc94ec 100644
Binary files a/blog/2024-07-09-verba-with-local-llm/img/verba-browser.png and b/blog/2024-07-09-verba-with-local-llm/img/verba-browser.png differ
diff --git a/blog/2024-07-09-verba-with-local-llm/img/verba-ollama-configured.png b/blog/2024-07-09-verba-with-local-llm/img/verba-ollama-configured.png
index 1411eff0d8..331e6d8dd0 100644
Binary files a/blog/2024-07-09-verba-with-local-llm/img/verba-ollama-configured.png and b/blog/2024-07-09-verba-with-local-llm/img/verba-ollama-configured.png differ
diff --git a/blog/2024-07-09-verba-with-local-llm/img/verba-rag-config.png b/blog/2024-07-09-verba-with-local-llm/img/verba-rag-config.png
index 0ea30beae9..2f5cde707d 100644
Binary files a/blog/2024-07-09-verba-with-local-llm/img/verba-rag-config.png and b/blog/2024-07-09-verba-with-local-llm/img/verba-rag-config.png differ
diff --git a/blog/2024-07-09-verba-with-local-llm/img/weaviate-cloud-console.png b/blog/2024-07-09-verba-with-local-llm/img/weaviate-cloud-console.png
index 31b41d2f82..156dd954a5 100644
Binary files a/blog/2024-07-09-verba-with-local-llm/img/weaviate-cloud-console.png and b/blog/2024-07-09-verba-with-local-llm/img/weaviate-cloud-console.png differ
diff --git a/blog/2024-07-17-js-june-recap/img/hero.png b/blog/2024-07-17-js-june-recap/img/hero.png
index 420c48f4e9..9da16a1abe 100644
Binary files a/blog/2024-07-17-js-june-recap/img/hero.png and b/blog/2024-07-17-js-june-recap/img/hero.png differ
diff --git a/blog/2024-07-17-js-june-recap/img/js-ts-weaviate.png b/blog/2024-07-17-js-june-recap/img/js-ts-weaviate.png
index 3bef2aa851..164f3d8e54 100644
Binary files a/blog/2024-07-17-js-june-recap/img/js-ts-weaviate.png and b/blog/2024-07-17-js-june-recap/img/js-ts-weaviate.png differ
diff --git a/blog/2024-07-17-js-june-recap/img/models.png b/blog/2024-07-17-js-june-recap/img/models.png
index 0718f68413..064842fa54 100644
Binary files a/blog/2024-07-17-js-june-recap/img/models.png and b/blog/2024-07-17-js-june-recap/img/models.png differ
diff --git a/blog/2024-07-17-js-june-recap/img/multimodal.png b/blog/2024-07-17-js-june-recap/img/multimodal.png
index 4fe4b55e4a..a32c17440f 100644
Binary files a/blog/2024-07-17-js-june-recap/img/multimodal.png and b/blog/2024-07-17-js-june-recap/img/multimodal.png differ
diff --git a/blog/2024-07-17-js-june-recap/img/rag.png b/blog/2024-07-17-js-june-recap/img/rag.png
index 3d9cdc5d1b..9dbaa908fe 100644
Binary files a/blog/2024-07-17-js-june-recap/img/rag.png and b/blog/2024-07-17-js-june-recap/img/rag.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/hero.png b/blog/2024-07-25-advanced-rag/img/hero.png
index c89bedf960..fe023d7ec8 100644
Binary files a/blog/2024-07-25-advanced-rag/img/hero.png and b/blog/2024-07-25-advanced-rag/img/hero.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image1.png b/blog/2024-07-25-advanced-rag/img/image1.png
index d3a4dd4993..977c55ae57 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image1.png and b/blog/2024-07-25-advanced-rag/img/image1.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image10.png b/blog/2024-07-25-advanced-rag/img/image10.png
index aa681c0440..138ecf35c6 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image10.png and b/blog/2024-07-25-advanced-rag/img/image10.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image11.png b/blog/2024-07-25-advanced-rag/img/image11.png
index 84c3dc5edf..2150bd7664 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image11.png and b/blog/2024-07-25-advanced-rag/img/image11.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image2.png b/blog/2024-07-25-advanced-rag/img/image2.png
index 8555d217bf..3b270f2e69 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image2.png and b/blog/2024-07-25-advanced-rag/img/image2.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image3.png b/blog/2024-07-25-advanced-rag/img/image3.png
index c7bb95e520..b1ef661591 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image3.png and b/blog/2024-07-25-advanced-rag/img/image3.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image4.png b/blog/2024-07-25-advanced-rag/img/image4.png
index e3612175a2..71092de0a8 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image4.png and b/blog/2024-07-25-advanced-rag/img/image4.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image5.png b/blog/2024-07-25-advanced-rag/img/image5.png
index d6b95d69f2..3f18d4d924 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image5.png and b/blog/2024-07-25-advanced-rag/img/image5.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image6.png b/blog/2024-07-25-advanced-rag/img/image6.png
index e34d7d2dda..474262a424 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image6.png and b/blog/2024-07-25-advanced-rag/img/image6.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image7.png b/blog/2024-07-25-advanced-rag/img/image7.png
index 63c7c979fc..dcff3cf84c 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image7.png and b/blog/2024-07-25-advanced-rag/img/image7.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image8.png b/blog/2024-07-25-advanced-rag/img/image8.png
index 9d5ef386ba..54a021cc21 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image8.png and b/blog/2024-07-25-advanced-rag/img/image8.png differ
diff --git a/blog/2024-07-25-advanced-rag/img/image9.png b/blog/2024-07-25-advanced-rag/img/image9.png
index 4d1852a7ed..39826b621b 100644
Binary files a/blog/2024-07-25-advanced-rag/img/image9.png and b/blog/2024-07-25-advanced-rag/img/image9.png differ
diff --git a/blog/2024-07-31-weaviate-1-26-release/img/Charts.jpg b/blog/2024-07-31-weaviate-1-26-release/img/Charts.jpg
index 43d9d190a3..0dbb15d0d7 100644
Binary files a/blog/2024-07-31-weaviate-1-26-release/img/Charts.jpg and b/blog/2024-07-31-weaviate-1-26-release/img/Charts.jpg differ
diff --git a/blog/2024-07-31-weaviate-1-26-release/img/Range.jpg b/blog/2024-07-31-weaviate-1-26-release/img/Range.jpg
index 95dbd44490..4ff7f8ab61 100644
Binary files a/blog/2024-07-31-weaviate-1-26-release/img/Range.jpg and b/blog/2024-07-31-weaviate-1-26-release/img/Range.jpg differ
diff --git a/blog/2024-07-31-weaviate-1-26-release/img/SQ.jpg b/blog/2024-07-31-weaviate-1-26-release/img/SQ.jpg
index 5c03803a39..57039f8b5f 100644
Binary files a/blog/2024-07-31-weaviate-1-26-release/img/SQ.jpg and b/blog/2024-07-31-weaviate-1-26-release/img/SQ.jpg differ
diff --git a/blog/2024-07-31-weaviate-1-26-release/img/hero.png b/blog/2024-07-31-weaviate-1-26-release/img/hero.png
index c79b16069e..81fb65e064 100644
Binary files a/blog/2024-07-31-weaviate-1-26-release/img/hero.png and b/blog/2024-07-31-weaviate-1-26-release/img/hero.png differ
diff --git a/blog/2024-08-22-zooviate/img/arch-diagram.png b/blog/2024-08-22-zooviate/img/arch-diagram.png
index 894ea8401b..d2f041f3c5 100644
Binary files a/blog/2024-08-22-zooviate/img/arch-diagram.png and b/blog/2024-08-22-zooviate/img/arch-diagram.png differ
diff --git a/blog/2024-08-22-zooviate/img/hero.png b/blog/2024-08-22-zooviate/img/hero.png
index 1f03525d57..79c61bf0c4 100644
Binary files a/blog/2024-08-22-zooviate/img/hero.png and b/blog/2024-08-22-zooviate/img/hero.png differ
diff --git a/blog/2024-08-22-zooviate/img/w-icon-vectorsearch-4.png b/blog/2024-08-22-zooviate/img/w-icon-vectorsearch-4.png
index 32296ce11c..2617308e10 100644
Binary files a/blog/2024-08-22-zooviate/img/w-icon-vectorsearch-4.png and b/blog/2024-08-22-zooviate/img/w-icon-vectorsearch-4.png differ
diff --git a/blog/2024-08-22-zooviate/img/wcd-screenshot.png b/blog/2024-08-22-zooviate/img/wcd-screenshot.png
index 334dd85b08..13a59d5644 100644
Binary files a/blog/2024-08-22-zooviate/img/wcd-screenshot.png and b/blog/2024-08-22-zooviate/img/wcd-screenshot.png differ
diff --git a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/Screenshot_2024-06-26_at_1.58.10_PM.png b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/Screenshot_2024-06-26_at_1.58.10_PM.png
deleted file mode 100644
index 33bc61a814..0000000000
Binary files a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/Screenshot_2024-06-26_at_1.58.10_PM.png and /dev/null differ
diff --git a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/Screenshot_2024-06-26_at_7.35.43_PM.png b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/Screenshot_2024-06-26_at_7.35.43_PM.png
deleted file mode 100644
index 6b6f63b689..0000000000
Binary files a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/Screenshot_2024-06-26_at_7.35.43_PM.png and /dev/null differ
diff --git a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/document-sample.png b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/document-sample.png
new file mode 100644
index 0000000000..a568fd8324
Binary files /dev/null and b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/document-sample.png differ
diff --git a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/hero.png b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/hero.png
index c781989809..e1645d7a46 100644
Binary files a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/hero.png and b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/hero.png differ
diff --git a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/summary.png b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/summary.png
new file mode 100644
index 0000000000..f92370906a
Binary files /dev/null and b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/img/summary.png differ
diff --git a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/index.mdx b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/index.mdx
index d2affefdba..17fa3c3f2b 100644
--- a/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/index.mdx
+++ b/blog/2024-09-03-enriching-and-ingesting-data-into-weaviate-with-aryn/index.mdx
@@ -159,7 +159,7 @@ This output is truncated for brevity
Each document in the DocSet now contains a list of elements that make up the document, including bounding boxes and text content. The following is a sample of a document after partitioning. Note that the table and figure are correctly identified, as are the individual cells in the table.
-
+
## Sycamore Transforms: Entity Extraction and Summarization
@@ -481,7 +481,7 @@ The summaries contain a text description of the image, as well as metadata about
Note that Sycamore leverages the structural information from the `partition` transform to improve the quality of the summary. For instance, for the following figure, the image contains only the bar chart, which does not have any axes. Sycamore recognizes that the caption is associated with the image and sends it to the LLM to aid with summarization. Because of this, the summary accurately indicates that the graphs show a distribution of house prices.
-
+
Summary:
diff --git a/blog/2024-09-05-late-chunking/img/chunking-strategies.png b/blog/2024-09-05-late-chunking/img/chunking-strategies.png
index 3955da5ac7..87153cc6aa 100644
Binary files a/blog/2024-09-05-late-chunking/img/chunking-strategies.png and b/blog/2024-09-05-late-chunking/img/chunking-strategies.png differ
diff --git a/blog/2024-09-05-late-chunking/img/goldilocks.png b/blog/2024-09-05-late-chunking/img/goldilocks.png
index 5d6969341b..788c585f15 100644
Binary files a/blog/2024-09-05-late-chunking/img/goldilocks.png and b/blog/2024-09-05-late-chunking/img/goldilocks.png differ
diff --git a/blog/2024-09-05-late-chunking/img/hero.png b/blog/2024-09-05-late-chunking/img/hero.png
index f3a6072cbc..aca787cba0 100644
Binary files a/blog/2024-09-05-late-chunking/img/hero.png and b/blog/2024-09-05-late-chunking/img/hero.png differ
diff --git a/blog/2024-09-10-python-v3-api-deprecation/index.mdx b/blog/2024-09-10-python-v3-api-deprecation/index.mdx
index 5c520ac9ec..f2cd0f98ee 100644
--- a/blog/2024-09-10-python-v3-api-deprecation/index.mdx
+++ b/blog/2024-09-10-python-v3-api-deprecation/index.mdx
@@ -39,7 +39,7 @@ Here's an illustrative example of the difference between the two versions. Note
text={PyCodeV3}
startMarker="# START SetVectorIndexParams"
endMarker="# END SetVectorIndexParams"
- language="py"
+ language="pyv3"
/>
diff --git a/blog/2024-09-17-better-ecommerce-typescript/img/final-application.png b/blog/2024-09-17-better-ecommerce-typescript/img/final-application.png
index 03af05f60b..08d6d3c639 100644
Binary files a/blog/2024-09-17-better-ecommerce-typescript/img/final-application.png and b/blog/2024-09-17-better-ecommerce-typescript/img/final-application.png differ
diff --git a/blog/2024-10-10-wealingo-demo/img/image.png b/blog/2024-10-10-wealingo-demo/img/image.png
index 8049efb9af..def7cf1bc9 100644
Binary files a/blog/2024-10-10-wealingo-demo/img/image.png and b/blog/2024-10-10-wealingo-demo/img/image.png differ
diff --git a/blog/2024-10-10-wealingo-demo/img/image1.png b/blog/2024-10-10-wealingo-demo/img/image1.png
index 0baceffd4b..545f8bdcd9 100644
Binary files a/blog/2024-10-10-wealingo-demo/img/image1.png and b/blog/2024-10-10-wealingo-demo/img/image1.png differ
diff --git a/blog/2024-10-10-wealingo-demo/img/image2.png b/blog/2024-10-10-wealingo-demo/img/image2.png
index 56e84c5c7a..5fad126089 100644
Binary files a/blog/2024-10-10-wealingo-demo/img/image2.png and b/blog/2024-10-10-wealingo-demo/img/image2.png differ
diff --git a/blog/2024-10-10-wealingo-demo/img/image3.png b/blog/2024-10-10-wealingo-demo/img/image3.png
index 46b11f52e3..457fb625ad 100644
Binary files a/blog/2024-10-10-wealingo-demo/img/image3.png and b/blog/2024-10-10-wealingo-demo/img/image3.png differ
diff --git a/blog/2024-10-10-wealingo-demo/img/image4.png b/blog/2024-10-10-wealingo-demo/img/image4.png
index fc31d453d3..4a0c09986b 100644
Binary files a/blog/2024-10-10-wealingo-demo/img/image4.png and b/blog/2024-10-10-wealingo-demo/img/image4.png differ
diff --git a/blog/2024-10-15-introduction-to-rag/img/hero.png b/blog/2024-10-15-introduction-to-rag/img/hero.png
index 24bd9ca71a..10266aeaba 100644
Binary files a/blog/2024-10-15-introduction-to-rag/img/hero.png and b/blog/2024-10-15-introduction-to-rag/img/hero.png differ
diff --git a/blog/2024-10-15-introduction-to-rag/img/rag-advanced-rag.png b/blog/2024-10-15-introduction-to-rag/img/rag-advanced-rag.png
index 1c1a50bc80..8bc1779484 100644
Binary files a/blog/2024-10-15-introduction-to-rag/img/rag-advanced-rag.png and b/blog/2024-10-15-introduction-to-rag/img/rag-advanced-rag.png differ
diff --git a/blog/2024-10-15-introduction-to-rag/img/rag-base.png b/blog/2024-10-15-introduction-to-rag/img/rag-base.png
index f8be67d1a0..25f6479757 100644
Binary files a/blog/2024-10-15-introduction-to-rag/img/rag-base.png and b/blog/2024-10-15-introduction-to-rag/img/rag-base.png differ
diff --git a/blog/2024-10-15-introduction-to-rag/img/rag-evaluation.png b/blog/2024-10-15-introduction-to-rag/img/rag-evaluation.png
index fc5b5c189b..bb7ea0c4ed 100644
Binary files a/blog/2024-10-15-introduction-to-rag/img/rag-evaluation.png and b/blog/2024-10-15-introduction-to-rag/img/rag-evaluation.png differ
diff --git a/blog/2024-10-15-introduction-to-rag/img/rag-stages.png b/blog/2024-10-15-introduction-to-rag/img/rag-stages.png
index c89697f802..b034291048 100644
Binary files a/blog/2024-10-15-introduction-to-rag/img/rag-stages.png and b/blog/2024-10-15-introduction-to-rag/img/rag-stages.png differ
diff --git a/blog/2024-10-15-introduction-to-rag/img/rag-workflow.png b/blog/2024-10-15-introduction-to-rag/img/rag-workflow.png
index e4fec016ad..1e73e3669e 100644
Binary files a/blog/2024-10-15-introduction-to-rag/img/rag-workflow.png and b/blog/2024-10-15-introduction-to-rag/img/rag-workflow.png differ
diff --git a/blog/2024-10-15-introduction-to-rag/index.mdx b/blog/2024-10-15-introduction-to-rag/index.mdx
index 25c329c007..16b565eee4 100644
--- a/blog/2024-10-15-introduction-to-rag/index.mdx
+++ b/blog/2024-10-15-introduction-to-rag/index.mdx
@@ -168,7 +168,7 @@ For a more in-depth exploration of this topic, check out our blog post on [advan
**AI agents** are autonomous systems that can interpret information, formulate plans, and make decisions. When added to a RAG pipeline, agents can reformulate user queries and re-retrieve more relevant information if initial results are inaccurate or irrelevant. Agentic RAG can also handle more complex queries that require multi-step reasoning, like comparing information across multiple documents, asking follow-up questions, and iteratively adjusting retrieval and generation strategies.
-To take a closer look at a RAG pipeline that incorporates agents and utilizes advanced techniques like text chunking and reranking, check out this [post](https://www.llamaindex.ai/blog/agentic-rag-with-llamaindex-2721b8a49ff6) and accompanying [notebook](https://github.com/cobusgreyling/LlamaIndex/blob/d8902482a247c76c7902ded143a875d5580f072a/Agentic_RAG_Multi_Document_Agents-v1.ipynb) on the LlamaIndex blog.
+To take a closer look at a RAG pipeline that incorporates agents, check out this [blog on Agentic RAG](/blog/what-is-agentic-rag/).
### Graph RAG
diff --git a/blog/2024-10-29-low-code-builder-lamatic/img/data-indexing-flow.png b/blog/2024-10-29-low-code-builder-lamatic/img/data-indexing-flow.png
new file mode 100644
index 0000000000..647eeaf896
Binary files /dev/null and b/blog/2024-10-29-low-code-builder-lamatic/img/data-indexing-flow.png differ
diff --git a/blog/2024-10-29-low-code-builder-lamatic/img/gen-ops-cycle.png b/blog/2024-10-29-low-code-builder-lamatic/img/gen-ops-cycle.png
new file mode 100644
index 0000000000..6fba904638
Binary files /dev/null and b/blog/2024-10-29-low-code-builder-lamatic/img/gen-ops-cycle.png differ
diff --git a/blog/2024-10-29-low-code-builder-lamatic/img/hero.png b/blog/2024-10-29-low-code-builder-lamatic/img/hero.png
new file mode 100644
index 0000000000..98eaa07d2b
Binary files /dev/null and b/blog/2024-10-29-low-code-builder-lamatic/img/hero.png differ
diff --git a/blog/2024-10-29-low-code-builder-lamatic/img/lamatic-value.png b/blog/2024-10-29-low-code-builder-lamatic/img/lamatic-value.png
new file mode 100644
index 0000000000..a6eae290e4
Binary files /dev/null and b/blog/2024-10-29-low-code-builder-lamatic/img/lamatic-value.png differ
diff --git a/blog/2024-10-29-low-code-builder-lamatic/img/retrieval-and-response-flow.png b/blog/2024-10-29-low-code-builder-lamatic/img/retrieval-and-response-flow.png
new file mode 100644
index 0000000000..0e384feb2c
Binary files /dev/null and b/blog/2024-10-29-low-code-builder-lamatic/img/retrieval-and-response-flow.png differ
diff --git a/blog/2024-10-29-low-code-builder-lamatic/index.mdx b/blog/2024-10-29-low-code-builder-lamatic/index.mdx
new file mode 100644
index 0000000000..118b2b0c02
--- /dev/null
+++ b/blog/2024-10-29-low-code-builder-lamatic/index.mdx
@@ -0,0 +1,126 @@
+---
+title: Build Contextual GenAI Apps in low code with Lamatic and Weaviate
+slug: low-code-builder-lamatic
+authors: [adam, aman]
+date: 2024-10-29
+tags: ['concepts']
+image: ./img/hero.png
+description: 'Learn about Retrieval Augmented Generation (RAG), including architecture, use cases, implementation, and evaluation.'
+---
+
+
+Large Language Models (LLMs) and Vector Databases have revolutionized knowledge management and automation. Retrieval pipelines now enable businesses with vast data repositories to provide their employees with instant access to crucial insights and answers. However, implementing these systems effectively requires a certain level of expertise. There's always a risk of misapplying AI, which is best avoided.
+
+Weaviate simplifies this process by offering an easy way to launch a serverless vector database instance and develop retrieval-based applications such as vector search and RAG chatbots. [Lamatic.ai](http://Lamatic.ai) takes this convenience a step further by providing serverless middleware that allows you to seamlessly integrate these features into your product. This approach eliminates the need for hiring specialized technical talent, extensive learning curves, and managing complex infrastructure.
+
+
+## About Lamatic.ai: Empowering High-Performance GenAI
+
+Lamatic is a comprehensive managed platform that combines a low-code visual builder, Weaviate, and integrations with various apps and models. It enables users to build, test, and deploy high-performance GenAI applications on the edge, all within minutes.
+
+Key offerings of [Lamatic.ai](http://Lamatic.ai) include:
+
+- **Intuitive Managed Backend:** Facilitates drag-and-drop connectivity to your preferred models, data sources, tools, and applications
+- **Collaborative Development Environment:** Enables teams to collaboratively build, test, and refine sophisticated GenAI workflows in a unified space
+- **Seamless Prototype to Production Pipeline:** Features weaviate and self-documenting GraphQL API that automatically scales on Serverless Edge infrastructure
+
+
+
+# Leveraging [Lamatic.ai](http://Lamatic.ai) and Weaviate for Effortless Contextual App Development
+
+[Lamatic.ai](http://Lamatic.ai) offers a powerful, fully managed GenAI stack that includes Weaviate. This innovative platform allows users to harness advanced AI technologies without requiring deep technical expertise. Let's explore how you can build a simple contextual chatbot using [Lamatic.ai](http://Lamatic.ai), following three key phases: Build, Connect, and Deploy.
+
+## Build: Creating Your GenAI Flows
+
+The first step is to build your flows. A flow is a visual representation of your GenAI app's logic. For our contextual chatbot, we'll need two flows:
+
+1. Data Indexing Flow:
+
+
+
+This flow automatically fetches files from a data source (like Google Drive) and indexes them into the VectorDB. It periodically checks for new changes and updates the index accordingly.
+
+
+
+2. Retrieval and Response Flow:
+
+
+
+This flow is triggered by a Slack command (e.g., /ask) and responds with answers based on the indexed data. It retrieves relevant information from the VectorDB and uses an LLM to generate a response.
+
+
+
+## Connect: Integrating Your Components
+
+After setting up your flows, connect the necessary components:
+
+- Apps: Integrate with Google Drive for data sourcing and Slack for user interaction.
+- Models: Choose and connect your preferred LLM for vectorization and querying.
+- Vector Store: Set up a Weaviate vector store within [Lamatic.ai](http://Lamatic.ai).
+
+## Deploy: Bringing Your Chatbot to Life
+
+Before deployment, thoroughly test your flows using [Lamatic.ai](http://Lamatic.ai)'s built-in debug options. Once you're satisfied with the performance, hit deploy to launch your project on the Edge. The indexing flow will start populating your Weaviate Store, and soon your Slack users can start asking questions using the /ask command.
+
+> The beauty of this approach is its versatility. With these same building blocks, you can create a wide range of applications that leverage contextual information from various sources. Let your imagination run wild! You can find more apps, templates and models here
+>
+>
+> [Lamatic.ai Hub](https://hub.lamatic.ai/)
+>
+
+---
+
+# 10 tips on building reliable Contextual apps with low code
+
+
+
+1. **Research Thoroughly:** Understand the problem domain and explore available technologies through academic papers, industry reports, and expert opinions.
+2. **Study Existing Cases:** Learn from successful implementations by examining case studies and engaging with professionals who've tackled similar challenges.
+3. **Plan Comprehensively:** Develop a detailed roadmap outlining project milestones, resources, risks, and contingencies before starting development.
+4. **Test Multiple Approaches:** Experiment with various strategies through prototypes, simulations, or A/B tests to find the most effective solution.
+5. **Handle Edge Cases:** Identify and resolve potential issues through comprehensive testing, including stress tests and unusual scenarios.
+6. **Seek User Feedback:** Engage with existing users to gather insights on relevance and usability through surveys, interviews, and usability tests.
+7. **Integrate Strategically:** Analyze your app's ecosystem to implement features where they provide the most value to users.
+8. **Automate Data Flows:** Create efficient, self-maintaining data pipelines with robust error handling and automated reporting.
+9. **Monitor Continuously:** Implement tracking systems to monitor performance and user interactions in real-time using analytics tools and custom dashboards.
+10. **Optimize Iteratively:** Refine your application based on user feedback, performance metrics, and market trends through regular review cycles.
+
+---
+
+# Potential Applications
+
+1. **Intelligent Documentation Assistant for Slack**
+Create a Slack bot that provides instant answers from your documentation, enhancing team productivity and knowledge sharing. For example, GenAI Collective significantly improved their event organizers' experience by implementing an answer bot within their apps, reducing response times and increasing efficiency.
+2. **Enhanced Website Search with Vector and Hybrid Capabilities**
+Use advanced search techniques to improve website navigation and content discovery, leading to a better user experience. For instance, [84000.co](http://84000.co) incorporated a hybrid search system to make Buddhist translations easily discoverable, combining traditional keyword search with AI-powered semantic understanding for more accurate results.
+3. **Integrated Image and Text Generation**
+Integrate GenAI features like image generation and content creation directly within your app. [Dinnerfy.com](http://Dinnerfy.com) exemplifies this by offering AI-generated recipe images and descriptions, enhancing user engagement and providing a more interactive culinary experience.
+4. **Automated Information Extraction and Summarization**
+Extract key information, score, and summarize large volumes of data. [Traq.ai](http://Traq.ai) demonstrates this capability by automatically summarizing and scoring customer calls, providing valuable insights for businesses to improve their customer service and sales strategies.
+
+# Go Beyond
+
+The examples above merely scratch the surface of what's achievable with [Lamatic.ai](http://Lamatic.ai). The possibilities are truly limitless. Rather than investing time and resources in recruiting scarce talent and managing complex infrastructure, focus on realizing your unique use case.
+We believe tools like [lamatic.ai](http://lamatic.ai) are pivotal in democratizing AI, making it more accessible to businesses of all sizes and ultimately improving lives. The low-code approach offered by [Lamatic.ai](http://Lamatic.ai) transforms complex technical concepts such as vector databases, text chunking, and model selection into intuitive, approachable processes. This simplification breaks down barriers that have traditionally deterred many from exploring AI applications.
+
+Furthermore, the platform's extensive library of templates and pre-built configurations significantly accelerates the development process. This allows businesses to rapidly prototype, iterate, and scale their AI solutions without getting bogged down in technical intricacies. By lowering the entry barrier and speeding up development cycles, [Lamatic.ai](http://Lamatic.ai) empowers organizations to focus on innovation and problem-solving rather than wrestling with infrastructure and code.
+
+The impact of such accessible AI tools extends beyond individual businesses. As more organizations leverage AI capabilities, we anticipate a ripple effect of innovation across industries, leading to enhanced products, services, and customer experiences. This democratization of AI technology has the potential to drive economic growth, foster competition, and ultimately contribute to societal progress on a global scale.
+
+Ready to explore further? Sign up for Lamatic AI or schedule a complimentary 1:1 AI workshop to discuss your specific use case.
+
+For those looking to bring Vector Database capabilities closer to their application infrastructure, get started with Weaviate Cloud today.
+
+
+
+import WhatsNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2024-10-29-weaviate-1-27-release/_core-1-27-include.mdx b/blog/2024-10-29-weaviate-1-27-release/_core-1-27-include.mdx
new file mode 100644
index 0000000000..889762b372
--- /dev/null
+++ b/blog/2024-10-29-weaviate-1-27-release/_core-1-27-include.mdx
@@ -0,0 +1,188 @@
+Weaviate `1.27` is here!
+
+Here are the release ⭐️*highlights*⭐️!
+
+
+
+- [**Big filtered search improvements**](#improved-filtered-search)
+- [**Multi-target vector search enhancement**](#multi-target-vector-search-enhancement)
+- [**Jina V3 embedding support**](#jina-v3-embedding-support)
+- [**Cancel backups**](#cancel-backups)
+- [**Other changes**](#other-changes)
+
+## Improved Filtered Search
+
+Weaviate `1.27` introduces a new filtering strategy with big benefits for performance and scalability.
+
+This exciting new search strategy/algorithm comes from our Applied Research team. Weaviate's new filtered search implementation is inspired by the popular [`ACORN`](https://arxiv.org/html/2403.04871v1) paper, improving on it to make it even better for Weaviate users.
+
+Weaviate's ACORN algorithm will particularly benefit those of you with very large datasets. More specifically, it should significantly improve performance challenges with negatively correlated filtered HNSW searches.
+
+:::tip What is a negatively correlated filtered search?
+
+A filtered search combines the structured, rule-based inclusion and exclusion of entities with the vector-based similarity search. The structured part is the filter, and the vector-based part is the search.
+
+
+In a negatively correlated filtered search, the filter disproportionately removes entities that would be most similar to the search query, as shown below.
+
+
+
+
+
+ An illustration of vector query / filter correlation (original from https://arxiv.org/html/2403.04871v1).
+
+
+
+This type of filtered search can be slower than other filtered searches. As in the figure, the vector search starts at the "wrong" end of the vector space, where the most similar entities are removed by the filter.
+
+
+The new ACORN filter strategy in Weaviate `1.27` dramatically improves the filtered vector search performance in situations like this.
+
+:::
+
+### How does ACORN work?
+
+The ACORN algorithm improves filtered search performance in three different ways:
+
+- Objects that do not meet the filters are ignored in distance calculations.
+- The algorithm reaches the relevant part of the HNSW graph faster, by using a multi-hop approach to evaluate the neighborhood of candidates.
+- The entry points are randomly seeded to speed up convergence to the filtered zone.
+
+This implementation is [Weaviate's own version](/developers/weaviate/concepts/filtering) of that described in the original paper, with modifications to make it even better for Weaviate users.
+
+### How to use ACORN
+
+ACORN can be enabled as the filter strategy by setting `filterStrategy` for the relevant HNSW vector index [in the collection configuration](/developers/weaviate/manage-data/collections#set-vector-index-parameters).
+
+:::note Note: Filter strategy parameter not yet supported by all clients
+At the time of writing, the `filterStrategy` parameter is not yet supported by all Weaviate clients. It will be added to all clients shortly.
+:::
+
+According to our internal tests, the ACORN algorithm generally improves the filtered vector search performances, with the most significant improvements in negatively correlated filtered searches.
+
+Note that there may be a small cost in recall, but our testing has found this to be minimal if not negligible.
+
+We are really excited for you to try it out.
+
+:::tip Who should use ACORN?
+We think that ACORN will benefit most users. But, we especially encourage you to try it out if you have experienced challenges with filtered HNSW searches.
+:::
+
+:::note Tell us about your experience ACORN
+We would love to hear from you about your experiences with ACORN. Please drop by to [our forum](https://forum.weaviate.io/t/acorn-feedback-mega-thread/7240) and tell us about it!
+:::
+
+## Multi-Target Vector Search Enhancement
+
+In Weaviate `1.27`, multi-target vector search has been improved further to allow searching the [same vector field multiple times](/developers/weaviate/search/multi-vector#specify-query-vectors) in one query.
+
+These new options provide even more flexibility for comprehensive searches through your data with Weaviate.
+
+Take a look at the [multi-target vector search page](/developers/weaviate/search/multi-vector) for more examples.
+
+:::info What is a multi-target vector search?
+
+Weaviate `1.26` introduced [multi-target vector searches](/developers/weaviate/search/multi-vector), where one search query can search across multiple target vectors.
+
+
+For example, you can search a product object with a query, and compare the query to the product's:
+- title vector and
+- description vector
+
+In one search!
+
+:::
+
+## Jina V3 embedding support
+
+Weaviate's [Jina AI embedding integration](/developers/weaviate/model-providers/jinaai/embeddings) now supports Jina AI's new V3 embeddings.
+
+### How to use the v3 embedding model
+
+From your perspective, all you have to set is the model name, to `jina-embeddings-v3` [here](/developers/weaviate/model-providers/jinaai/embeddings#configure-the-vectorizer).
+
+With the v3 model, you can optionally specify the `dimensions` parameter in the collection configuration with the Jina AI model integration. This will set Weaviate's Jina AI model integration to obtain vectors of the correct length. You can balance your search quality needs here with resource usage.
+
+Note you can combine Weaviate's [quantization techniques](/developers/weaviate/starter-guides/managing-resources/compression) with this to further manage resource usage.
+
+See the [Jina AI model integration page](/developers/weaviate/model-providers/jinaai/embeddings) for information on how to use this model.
+
+You can find more information about the model on [JinaAI's website](https://jina.ai/news/jina-embeddings-v3-a-frontier-multilingual-embedding-model/).
+
+:::info About Jina AI's v3 embedding model
+
+This model is a multilingual model that remains small and efficient while performing well on standard retrieval benchmarks.
+
+
+The v3 model adopts binary passage retrieval capabilities, which understands the difference between the database entries (`passages`) and the user `queries`.
+
+Weaviate automatically implements this for convenience, embedding objects as `passages` during import, and as `queries` during `nearText` search.
+
+
+Furthermore, the v3 model produces Matryoshka-style embeddings, This means that the generated embeddings can be truncated to suit the required precision.
+
+:::
+
+## Cancel backups
+
+[Creating backups](/developers/weaviate/configuration/backups) is a big part of database administration, especially in a production setting.
+
+In some cases, however, a backup process may be unwanted. Perhaps a backup was triggered by accident, or a backup process may be stuck.
+
+To address this, we have added a backup cancellation feature. Now, an unwanted, ongoing backup process can be stopped instead of consuming valuable CPU cycles.
+
+Backups can be cancelled [through a supported Weaviate client](/developers/weaviate/configuration/backups#cancel-backup), or through the [REST endpoint](/developers/weaviate/api/rest).
+
+## Other changes
+
+### Case sensitive vectorization
+
+:::info How does Weaviate vectorize text?
+Weaviate [integrates with embedding model providers](/developers/weaviate/model-providers) in order to conveniently vectorize data during import.
+:::
+
+From `v1.27`, text is no longer lowercased by default before vectorization.
+
+This change reflects the modern embeddings' capabilities to deal with uppercase (vs lowercase) characters. We expect that any changes to embeddings here will be very small, but we expect this to be closer to how modern embedding models expect inputs to look.
+
+If you prefer the previous behavior where the inputs are lowercased before import, set the `LOWERCASE_VECTORIZATION_INPUT` [environment variable](/developers/weaviate/config-refs/env-vars#general) to `true`.
+
+### Google modules renamed
+
+The Weaviate modules that underpin its [Google model integrations](/developers/weaviate/model-providers/google) have been renamed. Historically, they were named after the `PaLM` generation of models, thus having names `text2vec-palm`, `generative-palm`, `multi2vec-palm`.
+
+They have been renamed to `text2vec-google`, `generative-google`, `multi2vec-google` respectively.
+
+These modules are aliased under-the-hood in Weaviate, so your existing codebase will continue to work. However, we recommend updating your existing code if you are upgrading Weaviate so that your codebase follows the latest conventions.
+
+### PQ training set sampling
+
+[Product quantization (PQ)](/developers/weaviate/concepts/vector-quantization#product-quantization) is a technique used to compress vectors for efficient storage and retrieval.
+
+PQ requires calculation of centroids using a training set. From `1.27`, if you have ingested more objects than the training set size, Weaviate will now sample the training set from the ingested objects. This should make it more likely that the training set is representative of the data.
+
+### Reliability and performance improvements
+
+Between `1.26` and `1.27`, Weaviate's database and applied research teams have been hard at work, making significant improvements under-the-hood for reliability and performance improvements.
+
+We won't list them all here, but they include:
+- improvements to BM25 & hybrid query performance
+- data (LSM) store improvements,
+- cluster (Raft) improvements,
+- tombstone cleanup improvements
+
+and many more.
+
+As always, we recommend updating your version of Weaviate to the latest available patch release (e.g. `1.26.2` ➡️ `1.26.5`). And we encourage you to consider upgrading to the latest release (e.g. `1.26.2` ➡️ `1.27.0`).
+
+:::info Release notes
+For a full list of changes, see the [release notes for each version](https://github.com/weaviate/weaviate/tags).
+:::
+
+## Summary
+
+Enjoy the new features and improvements in Weaviate `1.27`. The release is available open-source as always [on GitHub](https://github.com/weaviate/weaviate/releases/tag/v1.27.0), and is available for new Sandboxes on [Weaviate Cloud](https://console.weaviate.cloud/). So please give it a try!
+
+It will be available for Serverless clusters on Weaviate Cloud soon as well.
+
+Thanks for reading, see you next time 👋!
diff --git a/blog/2024-10-29-weaviate-1-27-release/img/hero.png b/blog/2024-10-29-weaviate-1-27-release/img/hero.png
new file mode 100644
index 0000000000..a288638f65
Binary files /dev/null and b/blog/2024-10-29-weaviate-1-27-release/img/hero.png differ
diff --git a/blog/2024-10-29-weaviate-1-27-release/img/query-correlation.png b/blog/2024-10-29-weaviate-1-27-release/img/query-correlation.png
new file mode 100644
index 0000000000..c518437fc4
Binary files /dev/null and b/blog/2024-10-29-weaviate-1-27-release/img/query-correlation.png differ
diff --git a/blog/2024-10-29-weaviate-1-27-release/index.mdx b/blog/2024-10-29-weaviate-1-27-release/index.mdx
new file mode 100644
index 0000000000..0d15e312d8
--- /dev/null
+++ b/blog/2024-10-29-weaviate-1-27-release/index.mdx
@@ -0,0 +1,18 @@
+---
+title: Weaviate 1.27 Release
+slug: weaviate-1-27-release
+authors: [jp]
+date: 2024-10-29
+image: ./img/hero.png
+tags: ['release', 'engineering']
+description: "1.27 adds filtered search and multi-target vector improvements, Jina V3 embedding support and more!"
+
+---
+
+import Core127 from './_core-1-27-include.mdx';
+
+
+
+import WhatsNext from '/_includes/what-next.mdx'
+
+
diff --git a/blog/2024-11-05-what-is-agentic-rag/img/Components_of_an_AI_agent.png b/blog/2024-11-05-what-is-agentic-rag/img/Components_of_an_AI_agent.png
new file mode 100644
index 0000000000..f9ad6a1baa
Binary files /dev/null and b/blog/2024-11-05-what-is-agentic-rag/img/Components_of_an_AI_agent.png differ
diff --git a/blog/2024-11-05-what-is-agentic-rag/img/Multi_Agent_RAG_System.png b/blog/2024-11-05-what-is-agentic-rag/img/Multi_Agent_RAG_System.png
new file mode 100644
index 0000000000..226b096f02
Binary files /dev/null and b/blog/2024-11-05-what-is-agentic-rag/img/Multi_Agent_RAG_System.png differ
diff --git a/blog/2024-11-05-what-is-agentic-rag/img/ReAct.png b/blog/2024-11-05-what-is-agentic-rag/img/ReAct.png
new file mode 100644
index 0000000000..49fa126ec2
Binary files /dev/null and b/blog/2024-11-05-what-is-agentic-rag/img/ReAct.png differ
diff --git a/blog/2024-11-05-what-is-agentic-rag/img/Single_Agent_RAG_System_(Router).png b/blog/2024-11-05-what-is-agentic-rag/img/Single_Agent_RAG_System_(Router).png
new file mode 100644
index 0000000000..21629b01f3
Binary files /dev/null and b/blog/2024-11-05-what-is-agentic-rag/img/Single_Agent_RAG_System_(Router).png differ
diff --git a/blog/2024-11-05-what-is-agentic-rag/img/Vanilla_RAG.png b/blog/2024-11-05-what-is-agentic-rag/img/Vanilla_RAG.png
new file mode 100644
index 0000000000..1b0056765e
Binary files /dev/null and b/blog/2024-11-05-what-is-agentic-rag/img/Vanilla_RAG.png differ
diff --git a/blog/2024-11-05-what-is-agentic-rag/img/hero.png b/blog/2024-11-05-what-is-agentic-rag/img/hero.png
new file mode 100644
index 0000000000..423d82db9a
Binary files /dev/null and b/blog/2024-11-05-what-is-agentic-rag/img/hero.png differ
diff --git a/blog/2024-11-05-what-is-agentic-rag/index.mdx b/blog/2024-11-05-what-is-agentic-rag/index.mdx
new file mode 100644
index 0000000000..082ff3f3ef
--- /dev/null
+++ b/blog/2024-11-05-what-is-agentic-rag/index.mdx
@@ -0,0 +1,260 @@
+---
+title: What is Agentic RAG
+slug: what-is-agentic-rag
+authors: [erika, leonie]
+date: 2024-11-05
+tags: ['concepts']
+image: ./img/hero.png
+description: 'Learn about Agentic Retrieval Augmented Generation (RAG), including architecture, implementation, and and difference to vanilla RAG.'
+---
+
+
+
+While Retrieval-Augmented Generation (RAG) dominated 2023, [agentic workflows are driving massive progress in 2024](https://x.com/AndrewYNg/status/1770897666702233815?lang=en). The usage of AI agents opens up new possibilities for building more powerful, robust, and versatile Large Language Model(LLM)-powered applications. One possibility is enhancing RAG pipelines with AI agents in agentic RAG pipelines.
+
+This article introduces you to the concept of agentic RAG, its implementation, and its benefits and limitations.
+
+## Fundamentals of Agentic RAG
+
+Agentic RAG describes an AI agent-based implementation of RAG. Before we go any further, let’s quickly recap the fundamental concepts of RAG and AI agents.
+
+### What is Retrieval-Augmented Generation (RAG)
+
+[Retrieval-Augmented Generation (RAG)](https://weaviate.io/blog/introduction-to-rag) is a technique for building LLM-powered applications. It leverages an external knowledge source to provide the LLM with relevant context and reduce hallucinations.
+
+A naive RAG pipeline consists of a retrieval component (typically composed of an embedding model and a vector database) and a generative component (an LLM). At inference time, the user query is used to run a similarity search over the indexed documents to retrieve the most similar documents to the query and provide the LLM with additional context.
+
+
+
+Typical RAG applications have two considerable limitations:
+
+1. The naive RAG pipeline only considers one external knowledge source. However, some solutions might require two external knowledge sources, and some solutions might require external tools and APIs, such as web searches.
+2. They are a one-shot solution, which means that context is retrieved once. There is no reasoning or validation over the quality of the retrieved context.
+
+### What are Agents in AI Systems
+
+With the popularity of LLMs, new paradigms of AI agents and multi-agent systems have emerged. AI agents are LLMs with a role and task that have access to memory and external tools. The reasoning capabilities of LLMs help the agent plan the required steps and take action to complete the task at hand.
+
+Thus, the core components of an AI agent are:
+
+- LLM (with a role and a task)
+- Memory (short-term and long-term)
+- Planning (e.g., reflection, self-critics, query routing, etc.)
+- Tools (e.g., calculator, web search, etc.)
+
+
+
+One popular framework is the [ReAct framework](https://arxiv.org/abs/2210.03629). A ReAct agent can handle sequential multi-part queries while maintaining state (in memory) by combining routing, query planning, and tool use into a single entity.
+
+> ReAct = Reason + Act (with LLMs)
+
+
+The process involves the following steps:
+
+1. Thought: Upon receiving the user query, the agent reasons about the next action to take
+2. Action: the agent decides on an action and executes it (e.g., tool use)
+3. Observation: the agent observes the feedback from the action
+4. This process iterates until the agent completes the task and responds to the user.
+
+ 
+
+
+## What is Agentic RAG?
+
+Agentic RAG describes an AI agent-based implementation of RAG. Specifically, it incorporates AI agents into the RAG pipeline to orchestrate its components and perform additional actions beyond simple information retrieval and generation to overcome the limitations of the non-agentic pipeline.
+
+> Agentic RAG describes an AI agent-based implementation of RAG.
+
+### How does Agentic RAG work?
+
+Although agents can be incorporated in different stages of the RAG pipeline, agentic RAG most commonly refers to the use of agents in the retrieval component.
+
+Specifically, the retrieval component becomes agentic through the use of retrieval agents with access to different retriever tools, such as:
+
+- Vector search engine (also called a query engine) that performs vector search over a vector index (like in typical RAG pipelines)
+- Web search
+- Calculator
+- Any API to access software programmatically, such as email or chat programs
+- and many more.
+
+The RAG agent can then reason and act over the following example retrieval scenarios:
+
+1. Decide whether to retrieve information or not
+2. Decide which tool to use to retrieve relevant information
+3. Formulate the query itself
+4. Evaluate the retrieved context and decide whether it needs to re-retrieve.
+
+### Agentic RAG Architecture
+
+In contrast to the sequential naive RAG architecture, the core of the agentic RAG architecture is the agent. Agentic RAG architectures can have various levels of complexity. In the simplest form, a single-agent RAG architecture is a simple router. However, you can also add multiple agents into a multi-agent RAG architecture. This section discusses the two fundamental RAG architectures.
+
+**Single-Agent RAG (Router)**
+
+In its simplest form, agentic RAG is a router. This means you have at least two external knowledge sources, and the agent decides which one to retrieve additional context from. However, the external knowledge sources don't have to be limited to (vector) databases. You can retrieve further information from tools as well. For example, you can conduct a web search, or you could use an API to retrieve additional information from Slack channels or your email accounts.
+
+.png)
+
+**Multi-agent RAG Systems**
+
+As you can guess, the single-agent system also has its limitations because it's limited to only one agent with reasoning, retrieval, and answer generation in one. Therefore, it is beneficial to chain multiple agents into a multi-agent RAG application.
+
+For example, you can have one master agent who coordinates information retrieval among multiple specialized retrieval agents. For instance, one agent could retrieve information from proprietary internal data sources. Another agent could specialize in retrieving information from your personal accounts, such as email or chat. Another agent could also specialize in retrieving public information from web searches.
+
+
+
+### Beyond Retrieval Agents
+
+The above example shows the usage of different retrieval agents. However, you could also use agents for purposes other than retrieval. The possibilities of agents in the RAG system are manifold.
+
+## Agentic RAG vs. (Vanilla) RAG
+
+While the fundamental concept of RAG (sending a query, retrieving information, and generating a response) remains the same, **tool use generalizes it,** making it more flexible and powerful.
+
+Think of it this way: Common (vanilla) RAG is like being at the library (before smartphones existed) to answer a specific question. Agentic RAG, on the other hand, is like having a smartphone in your hand with a web browser, a calculator, your emails, etc.
+
+| | Vanilla RAG | Agentic RAG |
+| --- | --- | --- |
+| Access to external tools | No | Yes |
+| Query pre-processing | No | Yes |
+| Multi-step retrieval | No | Yes |
+| Validation of retrieved information | No | Yes |
+
+## Implementing Agentic RAG
+
+As outlined earlier, agents are comprised of multiple components. To build an agentic RAG pipeline, there are two options: a language model with function calling or an agent framework. Both implementations get to the same result, it will just depend on the control and flexibility you want.
+
+### Language Models with Function Calling
+
+Language models are the main component of agentic RAG systems. The other component is tools, which enable the language model access to external services. Language models with function calling offer a way to build an agentic system by allowing the model to interact with predefined tools. Language model providers have added this feature to their clients.
+
+In June 2023, [OpenAI released function calling](https://platform.openai.com/docs/assistants/tools/function-calling) for `gpt-3.5-turbo` and `gpt-4`. It enabled these models to reliably connect GPT’s capabilities with external tools and APIs. Developers quickly started building applications that plugged `gpt-4` into code executors, databases, calculators, and more.
+
+[Cohere](https://docs.cohere.com/v2/docs/tool-use) further launched their connectors API to add tools to the Command-R suite of models. Additionally, [Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/tool-use) and [Google](https://ai.google.dev/gemini-api/docs/function-calling) launched function calling for Claude and Gemini. By powering these models with external services, it can access and cite web resources, execute code and more.
+
+Function calling isn’t only for proprietary models. Ollama introduced tool support for popular open-source models like `Llama3.2`, `nemotron-mini`, and [others](https://ollama.com/search?c=tools).
+
+To build a tool, you first need to define the function. In this snippet, we’re writing a function that is using Weaviate’s [hybrid search](https://weaviate.io/developers/weaviate/search/hybrid) to retrieve objects from the database:
+
+```python
+def get_search_results(query: str) -> str:
+ """Sends a query to Weaviate's Hybrid Search. Parses the response into a {k}:{v} string."""
+
+ response = blogs.query.hybrid(query, limit=5)
+
+ stringified_response = ""
+ for idx, o in enumerate(response.objects):
+ stringified_response += f"Search Result: {idx+1}:\n"
+ for prop in o.properties:
+ stringified_response += f"{prop}:{o.properties[prop]}"
+ stringified_response += "\n"
+
+ return stringified_response
+```
+
+We will then pass the function to the language model via a `tools_schema`. The schema is then used in the prompt to the language model:
+
+```python
+tools_schema=[{
+ 'type': 'function',
+ 'function': {
+ 'name': 'get_search_results',
+ 'description': 'Get search results for a provided query.',
+ 'parameters': {
+ 'type': 'object',
+ 'properties': {
+ 'query': {
+ 'type': 'string',
+ 'description': 'The search query.',
+ },
+ },
+ 'required': ['query'],
+ },
+ },
+}]
+```
+
+Since you’re connecting to the language model API directly, you’ll need to write a loop that routes between the language model and tools:
+
+```python
+def ollama_generation_with_tools(user_message: str,
+ tools_schema: List, tool_mapping: Dict,
+ model_name: str = "llama3.1") -> str:
+ messages=[{
+ "role": "user",
+ "content": user_message
+ }]
+ response = ollama.chat(
+ model=model_name,
+ messages=messages,
+ tools=tools_schema
+ )
+ if not response["message"].get("tool_calls"):
+ return response["message"]["content"]
+ else:
+ for tool in response["message"]["tool_calls"]:
+ function_to_call = tool_mapping[tool["function"]["name"]]
+ print(f"Calling function {function_to_call}...")
+ function_response = function_to_call(tool["function"]["arguments"]["query"])
+ messages.append({
+ "role": "tool",
+ "content": function_response,
+ })
+
+ final_response = ollama.chat(model=model_name, messages=messages)
+ return final_response["message"]["content"]
+```
+
+Your query will then look like this:
+
+```python
+ollama_generation_with_tools("How is HNSW different from DiskANN?",
+ tools_schema=tools_schema, tool_mapping=tool_mapping)
+```
+
+You can follow along [this recipe](https://github.com/weaviate/recipes/blob/main/integrations/llm-frameworks/function-calling/ollama/ollama-weaviate-agents.ipynb) to recreate the above.
+
+### Agent Frameworks
+
+Agent Frameworks such as DSPy, LangChain, CrewAI, LlamaIndex, and Letta have emerged to facilitate building applications with language models. These frameworks simplify building agentic RAG systems by plugging pre-built templates together.
+
+- DSPy supports [ReAct](https://dspy-docs.vercel.app/deep-dive/modules/react/) agents and [Avatar](https://github.com/stanfordnlp/dspy/blob/main/examples/agents/avatar_langchain_tools.ipynb) optimization. Avatar optimization describes the use of automated prompt engineering for the descriptions of each tool.
+- [LangChain](https://www.langchain.com/) provides many services for working with tools. LangChain’s [LCEL](https://python.langchain.com/v0.1/docs/expression_language/) and [LangGraph](https://www.langchain.com/langgraph) frameworks further offer built-in tools.
+- [LlamaIndex](https://www.llamaindex.ai/) further introduces the QueryEngineTool, a collection of templates for retrieval tools.
+- [CrewAI](https://www.crewai.com/) is one of the leading frameworks for developing multi-agent systems. One of the key concepts utilized for tool use is sharing tools amongst agents.
+- [Swarm](https://github.com/openai/swarm) is a framework built by OpenAI for multi-agent orchestration. Swarm similarly focuses on how tools are shared amongst agents.
+- [Letta](https://docs.letta.com/introduction) interfaces reflecting and refining an internal world model as functions. This entails potentially using search results to update the agent’s memory of the chatbot user, in addition to responding to the question.
+
+## Why are Enterprises Adopting Agentic RAG
+
+Enterprises are moving on from vanilla RAG to building agentic RAG applications. [Replit released an agent](https://docs.replit.com/replitai/agent) that helps developers build and debug software. Additionally, [Microsoft announced copilots](https://blogs.microsoft.com/blog/2024/10/21/new-autonomous-agents-scale-your-team-like-never-before/) that work alongside users to provide suggestions in completing tasks. These are only a few examples of agents in production and the possibilities are endless.
+
+### Benefits of Agentic RAG
+
+The shift from vanilla RAG to agentic RAG allows these systems to produce more accurate responses, perform tasks autonomously, and better collaborate with humans.
+
+The benefit of agentic RAG lies primarily in the improved quality of retrieved additional information. By adding agents with access to tool use, the retrieval agent can route queries to specialized knowledge sources. Furthermore, the reasoning capabilities of the agent enable a layer of validation of the retrieved context before it is used for further processing. As a result, agentic RAG pipelines can lead to more robust and accurate responses.
+
+### Limitations of Agentic RAG
+
+However, there are always two sides to every coin. Using an AI agent a for subtask means incorporating an LLM to do a task. This comes with the limitations of using LLMs in any application, such as added latency and unreliability. Depending on the reasoning capabilities of the LLM, an agent may fail to complete a task sufficiently (or even at all). It is important to incorporate proper failure modes to help an AI agent get unstuck when they are unable to complete a task.
+
+## Summary
+
+This blog discussed the concept of agentic RAG, which involves incorporating agents into the RAG pipeline. Although agents can be used for many different purposes in a RAG pipeline, agentic RAG most often involves using retrieval agents with access to tools to generalize retrieval.
+
+This article discussed agentic RAG architectures using single-agent and multi-agent systems and their differences from vanilla RAG pipelines.
+
+With the rise and popularity of AI agent systems, many different frameworks are evolving for implementing agentic RAG, such as LlamaIndex, LangGraph, or CrewAI.
+
+Finally, this article discussed the benefits and limitations of agentic RAG pipelines.
+
+## Further Resources
+
+- [Notebook](https://github.com/weaviate/recipes/blob/main/integrations/llm-frameworks/function-calling/openai/openai-swarm.ipynb) using Swarm
+- [Notebook](https://github.com/weaviate/recipes/tree/main/integrations/llm-frameworks/letta) using Letta and Weaviate
+- [Notebooks](https://github.com/weaviate/recipes/tree/main/integrations/llm-frameworks/function-calling/ollama) on using function calling in Ollama
+- [Notebook](https://github.com/weaviate/recipes/blob/main/integrations/llm-frameworks/agentic-rag-benchmark/vanilla-rag-vs-agentic-rag.ipynb) on Vanilla RAG versus Agentic RAG
+
+import WhatsNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.2-dark.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.2-dark.png
new file mode 100644
index 0000000000..05aa704912
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.2-dark.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.2-light.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.2-light.png
new file mode 100644
index 0000000000..9abefd44d7
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.2-light.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.3-dark.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.3-dark.png
new file mode 100644
index 0000000000..f841c17d18
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.3-dark.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.3-light.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.3-light.png
new file mode 100644
index 0000000000..4cbfb21100
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.3-light.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.4-dark.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.4-dark.png
new file mode 100644
index 0000000000..f4320b3070
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.4-dark.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.4-light.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.4-light.png
new file mode 100644
index 0000000000..8e999aa153
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.4-light.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.5-dark.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.5-dark.png
new file mode 100644
index 0000000000..a28db5e445
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.5-dark.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.5-light.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.5-light.png
new file mode 100644
index 0000000000..21da971b08
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-0.5-light.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-correlated-0.2-dark.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-correlated-0.2-dark.png
new file mode 100644
index 0000000000..ea551b4ab5
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-correlated-0.2-dark.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-correlated-0.2-light.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-correlated-0.2-light.png
new file mode 100644
index 0000000000..540375e221
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-500k-filtered-dot-correlated-0.2-light.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-dot-filtered-negative-dark.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-dot-filtered-negative-dark.png
new file mode 100644
index 0000000000..2996ab8e39
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-dot-filtered-negative-dark.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-dot-filtered-negative-light.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-dot-filtered-negative-light.png
new file mode 100644
index 0000000000..cd26a1ca1e
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/beir-cohere-dot-filtered-negative-light.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/graph-example.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/graph-example.png
new file mode 100644
index 0000000000..5017d860e3
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/graph-example.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/hero-light.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/hero-light.png
new file mode 100644
index 0000000000..c8bf68194f
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/hero-light.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/img/hero.png b/blog/2024-11-19-speed-up-filtered-vector-search/img/hero.png
new file mode 100644
index 0000000000..f3784224fa
Binary files /dev/null and b/blog/2024-11-19-speed-up-filtered-vector-search/img/hero.png differ
diff --git a/blog/2024-11-19-speed-up-filtered-vector-search/index.mdx b/blog/2024-11-19-speed-up-filtered-vector-search/index.mdx
new file mode 100644
index 0000000000..203003e6cf
--- /dev/null
+++ b/blog/2024-11-19-speed-up-filtered-vector-search/index.mdx
@@ -0,0 +1,243 @@
+---
+title: How we speed up filtered vector search with ACORN
+slug: speed-up-filtered-vector-search
+authors: [abdel, john, jp]
+date: 2024-11-19
+tags: ['concepts', 'engineering', 'research']
+image: ./img/hero-light.png
+description: 'Learn about the challenges of filtered vector search and how Weaviate tackles them with ACORN.'
+---
+
+
+
+Vector embeddings have revolutionized how we think about search, especially across large datasets. Weaviate’s vector and hybrid search capabilities power recommendation engines, content management systems, and e-commerce sites.
+
+A “search”, however, is a complex animal. It is rare that a search query is as simple as “find items most similar to `comfortable dress shoes`. A query in the wild often requires that the resulting objects also meet certain criteria - like, `deliverable to my location`, `tan in color`, and `under $200`. In other words, they require traditional “filters” along with a “search”.
+
+This seemingly simple requirement creates a fascinating technical challenge.
+
+Do we filter first and then search for similarity? Search first and filter later? How does this affect the overall query quality and performance? Does the relationship between the search and filter criteria matter?
+
+In this post, we'll talk about how Weaviate has added a new filter strategy called ANN Constraint-Optimized Retrieval Network, commonly referred to as ACORN. ACORN is an innovative solution that we are very excited about. But before we get to all of that, let’s first explore these challenges at a high level, starting with the “pre-filtering” solution.
+
+## Pre-filtering for vector search
+
+One way to solve the filtered search problem is to apply the filter first over the data, and then perform the vector search. This is called a “pre-filtering” approach, and in its simplest form, is very straightforward. The search will determine which objects match the filter, skip the HNSW graph index, and just brute force these filtered objects.
+
+```mermaid
+flowchart LR
+%% Define main flow nodes
+Start["🔍 Search Query"] --> Filter["📑 Apply Filters"]
+Filter --> BruteForce["⚡ Brute Force Vector Search"]
+BruteForce --> Results["✨ Search Results"]
+
+%% Group the pre-filtering process
+subgraph prefilter ["Pre-filtering Search"]
+direction LR
+Filter
+BruteForce
+end
+
+%% Style nodes
+style Start fill:#ffffff,stroke:#B9C8DF,color:#130C49
+style Filter fill:#ffffff,stroke:#B9C8DF,color:#130C49
+style BruteForce fill:#ffffff,stroke:#B9C8DF,color:#130C49
+style Results fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+%% Style subgraph
+style prefilter fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+```
+
+Brute force over a small amount of data is actually quite fast especially if the filters reduce the dataset to a manageable number of objects. But this approach will quickly become slower as the number of filtered objects increase, as brute force query performance scales linearly with the dataset size. When using HNSW as the vector index in Weaviate, we use this approach for small filters (configurable by the `flatSearchCutOff` parameter).
+
+Now, what is a “smarter” solution for when you still have too many objects after applying the filter? In this case one option is post-filtering.
+
+## Post-filtering for vector search
+
+So, what is post-filtering? As the name suggests, post-filtering means we first search for the closest vectors to our query using the HNSW graph index, and then we apply the filter.
+
+```mermaid
+flowchart LR
+%% Define main flow nodes
+Start["🔍 Search Query"] --> VectorSearch["⚡ Vector Search"]
+VectorSearch --> Filter["📑 Apply Filters"]
+Filter --> Results["✨ Search Results"]
+
+%% Group the post-filtering process
+subgraph postfilter ["Post-filtering Search"]
+direction LR
+VectorSearch
+Filter
+end
+
+%% Style nodes
+style Start fill:#ffffff,stroke:#B9C8DF,color:#130C49
+style VectorSearch fill:#ffffff,stroke:#B9C8DF,color:#130C49
+style Filter fill:#ffffff,stroke:#B9C8DF,color:#130C49
+style Results fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+%% Style subgraph
+style postfilter fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+```
+
+This poses some questions immediately, however. Suppose we need to select the top 10 similar vectors to our query that includes a filter. We know that applying a filter over the results set will reduce its number. So, the search results prior to filtering must contain more results. But just how many more should it contain? Including more results will slow down the search, but not including enough may mean we end up with fewer than the desired number (10) once the filter is applied.
+
+We could estimate this based on the selectivity of the filter. For example, if we have a filter that roughly passes one objects out of 10, then we could obtain an unfiltered results set of 100 and expect to end up with 10 results post-filtering.
+
+However it is hard to estimate this accurately. A filter’s selectivity will vary, depending on the distribution of the filtered vectors and its correlation with the query vector.
+
+Imagine you have a filter on the price of the objects in an e-commerce dataset. We would expect that all else being equal, objects of similar prices would be more likely to be similar than those of vastly different prices. For example, consider an e-commerce site selling products with “ring” in the product name. Perhaps the site is, “Rings ‘r Us”. In this site’s vector index, diamond rings are likely to be closer in vector space as well as price, while novelty rings or even bike chain rings are likely to be in a different region in the search space.
+
+In other words, it is possible that your filter has correlation to your query. If you search for example for diamond rings with a high price filter, you might end up with a search result where most objects pass the filter. In this scenario, it will be a waste of performance to search for 100 objects in order to only select 10 in the filtering phase.
+
+In this high correlation situation the filter will pass a much larger portion of the results than 10%. Conversely, if we search for diamond rings also but with a very low price, chances are we will not obtain a result set where most objects will pass the filter. In this scenario, if we only fetch 100 objects in order to send 10 back, when we apply the filter we might end up without any objects at all.
+
+So in situations such as these, the algorithm designer could be faced with a choice, balancing the risk of insufficient results being found and achieving acceptable search performance.
+
+## Weaviate filtered search, pre-ACORN
+
+Up until the introduction of ACORN, Weaviate combined both solutions. As explained before, we use the `flatSearchCutOff` param to trigger switching between the two solutions. The post-filtering solution, however, is enhanced by filtering the result set inline while searching.
+
+This means that we explore the graph unfiltered but before we add candidates to the result set, we check whether each object passes the filter or not. In this way, we can dynamically identify the point where we need to stop our filtered search. We call this filter strategy `sweeping` and it is an effective approach, with some caveats.
+
+As mentioned before, traversing of the graph is performed without taking the filter into consideration. The drawback of this is that, especially with low correlation, it is possible to have situations where a significant percentage of objects do not pass the filter and so will never make it to the final result set. This causes unnecessary distance calculations which negatively impacts query performance.
+
+Can we just ignore those vectors for objects that will not pass the filter during the traversal of the graph? As it turns out - no, as we then risk disconnecting the graph, which breaks the connectivity needed for the HNSW search algorithm to greedily search through neighboring nodes. Disconnecting portions of the graph can dramatically reduce the result quality, as measured in recall.
+
+This is especially true when looking for cheap diamond rings with a very selective filter. Using a partly disconnected graph, it is possible that there is no path at all from the entry point in the graph for our query vector to the region of the search space where the results should be.
+
+As a result, the search performance (i.e. speed/throughput) can degrade even though HNSW is generally very fast. Given sufficiently large graphs, and restrictive enough filters with low correlation to the query vector, the search performance may start to be problematically slow.
+
+There exist some solutions that attempt to solve this problem. One example is the [Filtered DiskANN algorithm](https://harsha-simhadri.org/pubs/Filtered-DiskANN23.pdf). This solution aims to keep the graph connected for a predefined set of filters by adding some connections that would normally not exist in the graph. Although clever, it requires anticipating the filters and associated connections in advance, which may not even be possible sometimes. Imagine, again, the case where you have price involved. It would be impossible to anticipate all possible filters around it.
+
+Another alternative is [ACORN](https://arxiv.org/abs/2403.04871). We found ACORN much more appealing because it is filter-agnostic so doesn't require knowledge of what filters will be performed at index time.
+
+After learning about it, and [discussing](https://youtu.be/PxJ7FpbopKY) the topic with one of the authors of the paper, we decided this would be a good addition to Weaviate. We discuss the solution in detail next.
+
+## ACORN
+
+The original ACORN paper builds upon HNSW and provides two variants ACORN-1 and ACORN-γ. Our explanation will focus on ACORN-1 here which is the basis for Weaviate's implementation.
+
+The very first difference of ACORN from the normal HNSW algorithm is the elimination of normal triangle pruning. While the max connections parameter limits the graph size, ACORN avoids pruning to avoid disconnecting the graph for some filters. The graph requires compression given the different traversal approach, which we will cover after explaining the mechanism.
+
+The resulting graph is much larger without pruning. Once we filter our graph, however, to only consider connections where out-neighbors pass the filter criteria, the effective graph size will be manageable.
+
+The crucial difference is in how the neighborhood is expanded in ACORN to keep connectivity of the graph high. The main risk with filtering connections is that you could prune good paths as an intervening vector fails to pass the filter. As an example, consider the following:
+
+
+
+Let's assume we are searching for the top two closest neighbors of the blue query vector in this graph.
+
+The green dots illustrate vectors whose objects pass the filter, and the gray those that do not. Ideally, the search algorithm could follow the graph from an entry point to the target region. Starting from the red entry point, the graph would traverse to the vector labeled A, then to the vector labeled B as the distances reduce in that direction. From here, the algorithm should be able to find vector E through C and F through D. The problem is that if C and D are disconnected and thus not evaluated, then the search algorithm cannot find E and F. On the other hand, if considering C and D means we need to calculate the distance between the query vector and those vectors, going back to the above challenge of high resource usage.
+
+So, how does ACORN solve this issue? It cleverly uses a two-hop based expansion of the neighborhood, evaluating nodes that are two hops away rather than one. This means that E and F become out-neighbors of B, as they are two hops away. This speeds up traversal of a graph as you can imagine, and help to ameliorate challenges of low correlation between the filter and the query vector.
+
+Now let’s discuss how this strategy enables compression. Notice that vector A now has out-neighbors B and C, but C is also an out-neighbor of B. As a result there is no need to explicitly add C to the out-neighbors of A, as the B to C connection can be used to find C from A. We compress the connections of A by removing C from the list of A’s out-neighbors and only keep it on B.
+
+ACORN is a fascinating algorithm. If you would like to read more about it, we encourage you to refer to the [original paper](https://arxiv.org/abs/2403.04871). Our implementation of ACORN involved a few customizations - let’s touch on that before we wrap up.
+
+## Weaviate implementation of ACORN
+
+The Weaviate implementation of ACORN differs from that of the paper in a few ways.
+
+The first important change is during building the graph. While it is true that a different pruning logic helps keep the graph connected, we decided not to modify the indexing from “vanilla” HNSW.
+
+We did this mainly to limit resource overhead from unpruned nodes, and found the impact to recall acceptable. This came with the added benefit that for existing users of Weaviate, ACORN can be enabled without any requirement to reindex existing data.
+
+The second important difference is how we explore the graph while querying. Weaviate's ACORN implementation conditionally evaluates whether to use the two-hop expansion.
+
+If the first hop passes the filter, then the algorithm traverses the graph normally since the second hop will be explored anyway as the search progresses. We only use the two-hop expansion if the connecting node does not pass the filter. This means that when there is not a uniform distribution of the filtered nodes in the graph, our algorithm will operate as a regular HNSW in areas where a dense amount of nodes pass the filter and as ACORN where a sparse amount of nodes pass the filter.
+
+The third difference is how we seed additional entry points at layer zero of the graph to better deal with the problem of queries having low correlation to the filter.
+
+The challenge is that if we land in a region of the search space where barely any nodes pass the filter, we need to expand the search in all directions away from the query until we reach a region with nodes passing the filter.
+
+To better understand this, imagine you are looking for diamond rings around a low price point. In normal HNSW, the entrypoint on the base layer is influenced by distance calculations on higher layers so generally the entrypoint will land near the vector for “diamond rings”, however in this location very few nodes may match the price filter.
+
+The search will eventually find some low priced jewelry, but it can require the algorithm to search through a huge number of objects in the graph. The outcome is that the search performance will suffer. We found seeding additional entry points that match the filter to be very effective in improving performance here.
+
+## Internal test results - ACORN vs pre-ACORN
+
+We next show some experiments using for filtered vector search and differing correlation between the filters and the queries. You can easily test ACORN with your own data in Weaviate by [switching the filtering strategy in the collection definition to sweeping or acorn](/developers/weaviate/manage-data/collections#set-vector-index-parameters). The [benchmark tool](https://github.com/weaviate/weaviate-benchmarking) has been updated to easily test your data with the flags `-–filter` to signal that we are doing a filtered test, pre-filtering disabled with `--flatSearchCutoff` set to zero, and `-–filterStrategy` which accepts acorn as a valid strategy to indicate that we want to activate the new logic during querying.
+
+### Low correlation between filters and queries
+
+Here, we built collections out of the [Cohere's BEIR embeddings](https://huggingface.co/datasets/Cohere/beir-embed-english-v3). We merged different BEIR datasets together and used the source of data as the filter. This allows for us to construct both high and low correlated queries if the query vector has a filter matching or not matching its original dataset.
+
+Firstly, let's show some test results where we sample from the NQ and MSMARCO datasets, use query vectors from NQ but filter for MSMARCO and vary the percentage from 20% to 50%. The selectivity rate is the percentage of objects that pass the filter. The higher the selectivity rate, the more objects pass the filter and the easier it is to find the target region in the search space.
+
+The `acorn` line represents the new ACORN implementation, while the `sweeping` line represents the original Weaviate implementation. The y-axis shows the throughput in queries per second (QPS), while the x-axis shows the search quality (recall). The search quality is controlled by setting the `ef` parameter in the vector index configuration. The higher the `ef`, the better the search quality, but the slower the search.
+
+If we take a look at these figures, a very interesting pattern emerges. The `sweeping` implementation performs quite well at high selectivity rates, but the performance begins to drop as the selectivity rate decreases.
+
+**50% selectivity**
+
+
+
+
+**40% selectivity**
+
+
+
+
+**30% selectivity**
+
+
+
+
+**20% selectivity**
+
+
+
+
+When we look at the differences between graphs of the 50% and 20% selectivity rates, we can see that the relative positions between the two implementations are somewhat reversed. At 50% selectivity, the `sweeping` implementation performs better than the `acorn` implementation. However, at 20% selectivity, the `acorn` implementation is much faster than the `sweeping` implementation.
+
+For low selectivity rates (at 20%), it is worth noting that the query throughput (QPS) of the `sweeping` implementation can be half the throughput of the `acorn` implementation at the same recall level. Here, we start to see benefits of the ACORN algorithm's ability to maintain graph connectivity and traverse the graph more efficiently.
+
+Next, let's look at the results when the correlation between filters and queries are very low.
+
+### Very low correlation between filters and queries
+
+This is where the ACORN implementation really shows its value. In this example, we increased the number of datasets and kept low correlated filtering. This means that queries would target different datasets and thus we should expect a low correlation between the filters and queries. The results are shown in the image below.
+
+
+
+
+Under these settings, we can observe that Weaviate’s ACORN implementation significantly outperforms (an order of magnitude, in fact) the original sweeping algorithm. The difference here is such that the previous throughput (QPS) would have been a significant potential bottleneck, whereas our ACORN algorithm performs comparably to the previous scenario. The explanation for this disparity is that the old logic ends up performing nearly a brute force over the full graph while ACORN can find the target region much faster.
+
+You could imagine it makes little sense to have such a low correlation between filters and queries but in practice, this is actually not that unusual and just depends on the semantics of the filter.
+
+By preventing occurrences of such bottlenecks, Weaviate with the new ACORN search strategy is able to ensure that vector and hybrid search systems do not get bogged down when these “unusual” but sometimes quite common / real-world queries.
+
+### High correlation between filter and queries
+
+Finally let's show what happens with highly correlated queries using the same NQ and MSMARCO datasets as before.
+
+
+
+
+Here we see `sweeping` is still faster because due to the high correlation, the results from a filtered vs unfiltered vector search are very similar and the `sweeping` algorithm does very little extra calculations.
+
+To address these variations we are exploring a dynamic filter strategy. Currently as of Weaviate 1.27, the `acorn` filter strategy is opt-in and will automatically switch back to sweeping if it detects a high selectivity filter. We are looking further into dynamic selection of filtering strategy based on filter match rates while exploring the graph.
+
+## Conclusions
+
+Vector search with filtering is a common requirement in real-world applications, but achieving both high performance and accuracy can be challenging. This is especially so when filters and queries have low correlation. ACORN provides an elegant solution to this problem by maintaining graph connectivity through two-hop expansion, without requiring any special indexing or predefined filter configurations.
+
+ACORN presents a very interesting approach to improving filtered vector search on HNSW. Especially around low selectivity filters, the new search strategy is able to ensure that a vector/hybrid search system performs well regardless of the filter.
+
+Our implementation in Weaviate builds on ACORN's strengths while adding practical optimizations:
+
+- No re-indexing required - works with existing HNSW indexes
+- Adaptive two-hop expansion that switches between HNSW and ACORN based on filter density
+- Smart handling of entry points for better performance
+
+The results speak for themselves: up to 10x performance improvement in challenging scenarios where filters and queries have low correlation, while maintaining good performance in other cases. This makes filtered vector search more robust and predictable.
+
+Ready to try it out? ACORN is available in [Weaviate 1.27](/blog/weaviate-1-27-release/) and newer. Simply enable it in your collection configuration and start experimenting with your own datasets. We're excited to hear about your experiences and use cases!
+
+For more information on using ACORN in Weaviate, check out our [documentation](/developers/weaviate/), such as [how to enable ACORN](/developers/weaviate/manage-data/collections#set-vector-index-parameters), and [how to perform filtered searches](/developers/weaviate/search/filters).
+
+import WhatsNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/img/ann-annoy.png b/blog/2024-11-21-vector-search-explained/img/ann-annoy.png
similarity index 100%
rename from blog/2022-09-13-why-is-vector-search-so-fast/img/ann-annoy.png
rename to blog/2024-11-21-vector-search-explained/img/ann-annoy.png
diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/img/ann-hnsw.png b/blog/2024-11-21-vector-search-explained/img/ann-hnsw.png
similarity index 100%
rename from blog/2022-09-13-why-is-vector-search-so-fast/img/ann-hnsw.png
rename to blog/2024-11-21-vector-search-explained/img/ann-hnsw.png
diff --git a/blog/2024-11-21-vector-search-explained/img/any2any_L.png b/blog/2024-11-21-vector-search-explained/img/any2any_L.png
new file mode 100644
index 0000000000..2840bc0873
Binary files /dev/null and b/blog/2024-11-21-vector-search-explained/img/any2any_L.png differ
diff --git a/blog/2024-11-21-vector-search-explained/img/hero.png b/blog/2024-11-21-vector-search-explained/img/hero.png
new file mode 100644
index 0000000000..ee349c813a
Binary files /dev/null and b/blog/2024-11-21-vector-search-explained/img/hero.png differ
diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/img/knn-boules.png b/blog/2024-11-21-vector-search-explained/img/knn-boules.png
similarity index 100%
rename from blog/2022-09-13-why-is-vector-search-so-fast/img/knn-boules.png
rename to blog/2024-11-21-vector-search-explained/img/knn-boules.png
diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/img/knn-linear.png b/blog/2024-11-21-vector-search-explained/img/knn-linear.png
similarity index 100%
rename from blog/2022-09-13-why-is-vector-search-so-fast/img/knn-linear.png
rename to blog/2024-11-21-vector-search-explained/img/knn-linear.png
diff --git a/blog/2024-11-21-vector-search-explained/img/rag.png b/blog/2024-11-21-vector-search-explained/img/rag.png
new file mode 100644
index 0000000000..fcf447b4a0
Binary files /dev/null and b/blog/2024-11-21-vector-search-explained/img/rag.png differ
diff --git a/blog/2024-11-21-vector-search-explained/img/vector-search.png b/blog/2024-11-21-vector-search-explained/img/vector-search.png
new file mode 100644
index 0000000000..9f2ab1ff54
Binary files /dev/null and b/blog/2024-11-21-vector-search-explained/img/vector-search.png differ
diff --git a/blog/2024-11-21-vector-search-explained/index.mdx b/blog/2024-11-21-vector-search-explained/index.mdx
new file mode 100644
index 0000000000..3f57916206
--- /dev/null
+++ b/blog/2024-11-21-vector-search-explained/index.mdx
@@ -0,0 +1,297 @@
+---
+title: Vector Search Explained
+slug: vector-search-explained
+authors: [victoria]
+date: 2024-11-21
+tags: ['concepts']
+image: ./img/hero.png
+description: "Learn about vector search, a technique that uses mathematical representations of data to find similar items in large data sets."
+---
+
+
+
+Vector search has fundamentally shifted how we approach information retrieval in modern development. It operates on numerical representations of data called vector embeddings, which capture the semantic meaning in a vector space. Vector search can identify related objects without requiring exact text matches, making it increasingly important for modern information retrieval systems, particularly in AI-native applications where traditional search systems fall short.
+
+## What is vector search?
+
+Vector search is a technique for finding and retrieving similar items in large datasets by comparing their vector representations, which are numerical encodings of their features. Unlike traditional search that relies on exact matches, vector search looks for similarity based on meaning or context. It's used in applications like image retrieval, recommendation systems, and search engines.
+
+## How does vector search work?
+
+Vector search works by converting data and queries into [vector embeddings](https://weaviate.io/blog/vector-embeddings-explained). Vector embeddings are generated by language models that learn to capture the meaning and context of data within numerical representations.
+
+
+
+At the time of entry/import (or any significant changes to data objects), every data object, and the query, are converted into numerical vector representations using [embedding models](https://weaviate.io/blog/how-to-choose-an-embedding-model). Every data object in a dataset gets a vector, and this is compared to the query vector at search time.
+
+In a nutshell, vector embeddings are an array of numbers, which can be used as coordinates in a high-dimensional space. Although it is hard to imagine coordinates in more than 3-dimensional space (x, y, z), we can still use the vectors to compute the distance between vectors, which can be used to indicate similarity between objects. There are many different [distance metrics](https://weaviate.io/blog/distance-metrics-in-vector-search), like [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) and [Euclidean distance (L2 distance)](https://en.wikipedia.org/wiki/Euclidean_distance).
+
+Whenever we run a query (like: "What is the tallest building in Berlin?"), a vector search system will convert it to a "query" vector. The task of a vector database is to identify and retrieve a list of vectors that are closest to the vector of your query, using a distance metric and a search algorithm.
+
+This is a bit like a game of boules – where the small marker (jack) is the location of our query vector, and the balls (boules) are our data vectors – and we need to find the boules that are nearest to the marker.
+
+An example of a search algorithm is a [k-nearest neighbors (kNN) algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm), which returns the k nearest vectors, by calculating a similarity score for every data vector in the database to the query vector. In our boules example, with 6 boules, the kNN algorithm would measure the distance between the jack and each of the 6 boules on the ground. This would result in 6 separate calculations.
+
+
+
+*[kNN search in a game of Boules.]*
+
+## How to implement Vector Search (from scratch) in Python
+
+We can implement a simple vector search solution from scratch with just a few lines of code in Python.
+
+Let’s start by installing the OpenAI python package to generate vector embeddings for our text, and numpy to do our similarity calculations.
+
+```
+pip install openai
+pip install numpy
+```
+
+We can then import all our dependencies, set our \`OPENAI\_API\_KEY\`, and define our list of sentences.
+
+```py
+from openai import OpenAI
+from dotenv import load_dotenv
+import os
+import numpy as np
+
+load_dotenv()
+
+client = OpenAI(
+ api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+sentences = [
+ "Best pizza places nearby.",
+ "Popular breakfast spots in New York.",
+ "Top-rated seafood restaurants in Miami.",
+ "Cheap hotels near the beach.",
+ "Recipes for quick pasta dishes.",
+]
+```
+
+We can use OpenAI to generate vector embeddings for every sentence, and store these in a new dictionary.
+
+```py
+def get_embedding(text, model="text-embedding-3-small"):
+ embedding = client.embeddings.create(input=[text], model=model).data[0].embedding
+ return embedding
+
+sentence_vectors = {}
+for sentence in sentences:
+ embedding = get_embedding(sentence)
+ sentence_vectors[sentence] = embedding
+
+```
+
+To calculate the similarity score between the query and each sentence, we can use [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) scoring method.
+
+```py
+def calculate_cosine_similarity(query_vector, vector):
+ return np.dot(query_vector, vector) / (np.linalg.norm(query_vector) * np.linalg.norm(vector))
+```
+
+Finally, this function will take our query, convert it to a vector, and calculate the similarity score between every query vector and document vector. Then, it will order the results based on relevance, and return the top two scored sentences.
+
+```py
+def get_top_n_similar(query_sentence, n=2):
+ query_embedding = get_embedding(query_sentence)
+
+ similarities = {sentence: calculate_cosine_similarity(query_embedding, sentence_vectors[sentence]) for sentence in sentences}
+
+ sorted_similarities = dict(sorted(similarities.items(), key=lambda x: x[1], reverse=True))
+ top_matches = list(sorted_similarities.items())[:n]
+
+ for sentence, score in top_matches:
+ print(f"Similarity: {score:.4f} - {sentence}")
+```
+
+You can see that even though the query and the first returned sentence don’t have all the words in common, it still scores as a high match because the *meaning* is very similar.
+
+```py
+query_sentence = "Find the best pizza restaurant close to me."
+get_top_n_similar(query_sentence, n=2)
+```
+
+```
+Similarity: 0.7056 - Best pizza places nearby.
+Similarity: 0.3585 - Top-rated seafood restaurants in Miami.
+```
+
+Although this method is able to give us similar items based on text vectors, it’s pretty inefficient. Manually computing cosine similarity over every vector in a large dataset can become computationally expensive fast. Also, without an indexing mechanism, all the vectors are stored in their raw form, requiring high memory consumption and slowing down search speeds. And finally, just storing the vectors and text objects in a dictionary means we lack structured database features.
+
+To fix all these problems, we can use a vector database to store and search through our vectors.
+
+## Vector search in vector databases
+
+Vector search, also called semantic search, using a vector database is able to handle large amounts of [unstructured data](https://weaviate.io/learn/knowledgecards/unstructured-data-objects) (think paragraphs instead of spreadsheets) quickly and efficiently, and provides relevant results in search and recommendation systems based on semantic similarity rather than exact matches. This can allow you to do advanced search through millions of documents, for example, [a Wikipedia dataset with over 28 million paragraphs](https://huggingface.co/datasets/weaviate/wiki-sample).
+
+If we query for articles related to: "urban planning in Europe", the vector database (like [Weaviate](https://weaviate.io/developers/weaviate)) responds with a series of articles about the topic, such as "The cities designed to be capitals".
+
+Finding the correct answer in a gigantic repository of unstructured data is not the most impressive part of [vector databases](https://weaviate.io/blog/what-is-a-vector-database) (I mean, it is very impressive), but it is the 🚀 speed at which it all happens. To find the most relevant answer to our semantic search query, it only takes milliseconds in a dataset containing 28 million paragraphs. This is super important when dealing with massive amounts of data, like in recommendation engines or large document datasets.
+
+The inevitable question that follows up this explanation is always: Why is this so incredibly fast?
+
+### Approximate nearest neighbors (ANN)
+
+Instead of comparing vectors one by one, most vector databases use [Approximate Nearest Neighbor (ANN) algorithms](https://weaviate.io/learn/knowledgecards/ann-approximate-nearest-neighbor), which trade off a bit of accuracy (hence the A in the name) for a huge gain in speed.
+
+ANN algorithms may not return the true k nearest vectors, but they are very efficient. ANN algorithms maintain good performance (sublinear time, e.g. (poly)logarithmic complexity) on very large-scale datasets.
+
+
+
+*[O(n) and O(log n) complexity]*
+
+Note that most vector databases allow you to configure how your ANN algorithm should behave. This lets you find the right balance between the recall tradeoff (the fraction of results that are the true top-k nearest neighbors), latency, throughput (queries per second) and import time.
+
+### Examples of ANN algorithms
+
+Examples of ANN methods are:
+
+* trees – e.g. [ANNOY](https://github.com/spotify/annoy) (Figure 3),
+* proximity graphs \- e.g. [HNSW](https://arxiv.org/abs/1603.09320) (Figure 4),
+* clustering \- e.g. [FAISS](https://github.com/facebookresearch/faiss),
+* hashing \- e.g. [LSH](https://en.wikipedia.org/wiki/Locality-sensitive_hashing)
+
+
+*[Tree-based ANN search]*
+
+Which algorithm works best depends on your project. Performance can be measured in terms of latency, throughput (queries per second), build time, and accuracy (recall). These four components often have a tradeoff, so it depends on the use case which method works best.
+So, while ANN is not some magic method that will always find the true k nearest neighbors in a dataset, it can find a pretty good approximation of the true k neighbors. And it can do this in a fraction of the time!
+
+### HNSW in Weaviate
+
+[Weaviate](https://weaviate.io/developers/weaviate) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The ANN algorithm that Weaviate uses is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](https://weaviate.io/developers/weaviate/concepts/vector-index#hnsw).
+
+
+
+*[HNSW - Proximity graph-based ANN search]*
+
+HNSW works by organizing vectors into a hierarchical, multi-layered graph structure, which allows for fast navigation through the dataset during search. The structure of HNSW balances longer distances for faster search in upper layers and shorter distances for accurate search in lower layers.
+
+In Weaviate's implementation, HNSW is enhanced to support full [CRUD operations](https://weaviate.io/blog/crud-support-in-weaviate) and allows for real-time querying, updates, and deletions, with features like incremental disk writes for crash recovery and [asynchronous cleanup processes](https://github.com/nmslib/hnswlib/issues/4#issuecomment-678315156) for maintaining index freshness.
+
+Check out [Weaviate ANN benchmarks](https://weaviate.io/developers/weaviate/benchmarks/ann) to see how HNSW performed on realistic large-scale datasets. You can use it to compare the tradeoffs between recall, QPS, latency, and import time.
+
+You will find it interesting to see that Weaviate can maintain very high recall rates (\>95%), whilst keeping high throughput and low latency (both in milliseconds). That is exactly what you need for fast, but reliable vector search\!
+
+If you’re interested in benchmarking for your own dataset, [check out this webinar](https://events.weaviate.io/benchmarking-webinar).
+
+### ANN vs. KNN
+
+kNN, or [k-nearest neighbors (kNN) algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm), differs from ANN because it calculates a similarity score for *every data vector* in the database compared to the query vector, much like our vector search from scratch example above
+
+Comparing a query vector with 10, 100, or 1000 data vectors in just two dimensions is an easy job. But of course, in the real world, we are more likely to deal with millions (like in the Wikipedia dataset) or even billions of data items. In addition, the number of dimensions that most embedding models use in semantic search goes up to hundreds or thousands of dimensions\!
+
+The *brute force* of a kNN search is computationally very expensive \- and depending on the size of your database, a single query could take anything from several seconds to even hours (yikes 😅). If you compare a vector with 300 dimensions with 10M vectors, the search system would need to do 300 x 10M = 3B computations\! The number of required calculations increases linearly with the number of data points (O(n)).
+
+In summary, kNN search doesn't scale well, and it is hard to imagine using it with a large dataset in production.
+
+## Types of vector search
+
+Vector search is not just limited to text, or even text in a single language. Anything can be converted into a vector with the right embedding model, whether it’s images, audio, video, or multi-lingual documents. This means we can develop multi-modal or multi-lingual semantic search systems that can handle a variety of data formats, languages, or search types.
+
+### Image Vector Search
+
+[Image vector search](https://weaviate.io/developers/weaviate/search/image) converts images into vector representations to enable similarity searches between images. The vector embeddings encode features like colors, shapes, and textures so that you can search for images based on visual similarity rather than metadata alone. This type of search is frequently used in fields like e-commerce for finding visually similar products, or in content moderation.
+
+### Audio Vector Search
+
+Audio vector search transforms audio files into vectors to power similarity search based on sound characteristics, such as tone, rhythm, or melody. This is used in applications like music discovery platforms, sound effect libraries, and voice recognition systems.
+
+### Video Vector Search
+
+Video vector search converts videos into vector embeddings through methods like sampling frames or analyzing scene features for search based on visual and sometimes audio similarity. This method is popular in applications like content libraries, surveillance, and media databases.
+
+### Multimodal Vector Search
+
+
+
+[Multimodal vector search](https://weaviate.io/developers/academy/py/starter_multimodal_data/mm_searches/multimodal) combines different data types—like text, images, and audio—in the same vector space for cross-type comparisons. For example, you could find images or audio most similar to a text query, like retrieving images of a lion or a lion’s roar based on a description alone. This can be used in applications to search across various media formats, like e-commerce, digital asset management, and social media platforms.
+
+### Multilingual Vector Search
+
+Multilingual vector search can compare text across languages by embedding it into the same vector space. For instance, a query in English could retrieve documents, captions, or content in other languages, such as French or Chinese, describing similar concepts. These cross-lingual searches can power applications like global e-commerce, multilingual customer support, and international content discovery.
+
+### Hybrid Search
+
+[Hybrid search](https://weaviate.io/blog/hybrid-search-explained) combines multiple search types, usually vector search and keyword search, in a single system. By combining traditional keyword search with semantic search, we get the best of both worlds: semantic context \+ specific keywords. Semantic search works great for understanding the general meaning of the documents and query, but it doesn’t prioritize exact matches such as names, industry-specific jargon, or rare words, which can be helpful in many types of applications. But without semantic search (only keyword), the results can miss relevant semantic information and context about the desired topic. Combining these two methods can improve both the accuracy and relevance of search results.
+
+## Benefits of vector search
+
+[Traditional keyword-based search systems](https://weaviate.io/learn/knowledgecards/keyword-search) can only match *exact terms or phrases*, or key words, in text. In contrast, vector search returns similar items based on their underlying context or meaning, and works with a variety of data types, including text, images, audio or video. For example, a search for "healthy snacks" might return semantically related terms like "nutritious food" or "low-calorie treats", or in the case of a [multimodal system](https://weaviate.io/learn/knowledgecards/multimodal-embeddings-models), pictures of granola bars or fruit.
+
+This is why vector search is often also called semantic search. It enables a more “human-like” search experience, allowing users to find related objects without needing to have the exact right words in their query.
+
+## Vector search use cases
+
+Vector search can power a variety of different applications and use cases, from advanced search systems, to recommendation systems, to chatbots.
+
+### Search Systems
+
+Vector search can be applied to a wide range of different search systems, from billion-scale e-commerce applications to multi-modal and multi-lingual search apps to internal document search in large enterprises. For example, in e-commerce, semantic search can recommend products based on customer intent, even if they don't have the exact keywords in their search query. With the right embedding model selection, you could create [multilingual](https://weaviate.io/blog/weaviate-non-english-languages), like [WeaLingo](https://weaviate.io/blog/wealingo-demo), or [multimodal](https://weaviate.io/developers/academy/py/starter_multimodal_data/mm_searches/multimodal) search apps out of the box. Vector search can also speed up result time for use cases like enterprise search, because the ANN algorithms allow for such fast retrieval speeds over large numbers of unstructured documents.
+
+### Recommendation Systems
+
+Vector search can be used in [recommendation systems](https://weaviate.io/workbench/recommender) to recommend similar products, movies, or content based on user preferences by finding items with similar vector representations, even without shared metadata or tags. This is widely used in social media apps, news sites, or e-commerce stores.
+
+### Retrieval Augmented Generation (RAG)
+
+[Retrieval Augmented Generation](https://weaviate.io/blog/introduction-to-rag), or RAG, is a popular use case of vector search used in applications like chatbots or question-answering systems. RAG is just vector search with an extra added step \- the similar results returned by the vector database are given to a large language model (LLM) to generate a contextually relevant response to the user’s query. RAG helps minimize model hallucinations, increase accuracy of responses, and allows generative models to access specialized knowledge to answer complex, data-driven queries.
+
+Semantic search is great for RAG applications because of its speed and ability to search for documents based on meaning, rather than exact matches.
+
+
+
+## Vector search solutions
+
+There are a few different ways to enable vector search in applications:
+
+[**Vector indexing libraries**](https://weaviate.io/blog/vector-library-vs-vector-database#vector-libraries) such as [FAISS](https://github.com/facebookresearch/faiss), [Annoy](https://github.com/spotify/annoy), and [ScaNN](https://github.com/google-research/google-research/tree/master/scann), are optimized for in-memory similarity searches and only store vectors, not the data objects they originate from. Their indexes are generally immutable, meaning they can’t be updated dynamically without rebuilding. Additionally, vector libraries often require all data to be imported before querying, which can limit their flexibility in dynamic or constantly updating environments. These libraries are good for applications that involve static data and do not need full CRUD operations or persistence capabilities.
+
+**Vector-capable databases** extend traditional databases with vector search capabilities, allowing businesses to incorporate semantic search while leveraging existing database features. These solutions generally struggle with reliability and speed at scale.
+
+[**Vector databases**](https://weaviate.io/blog/what-is-a-vector-database) (such as Weaviate) offer a comprehensive solution to semantic search use cases, supporting vector indexing and also managing data persistence, scaling, and integration with the AI ecosystem. They have flexible solutions for a variety of use cases, from AI applications at scale to users who are just getting started.
+
+A **vector search engine** is often used interchangeably with a vector database, but they are technically different: a vector search engine focuses only on the retrieval layer, while a vector database includes additional features like storage, data management, and clustering.
+
+## Vector search FAQs
+
+### Semantic search vs Vector search
+
+In terms of formal definitions, vector search is just the process of arranging vector embeddings, or vectors, into a vector index to perform similarity searches, while semantic search builds on the basic definition of vector search to return more relevant results based on the meaning of text rather than exact terms. In practice, though, vector search and semantic search are often used interchangeably.
+
+### Vector search vs Keyword search
+
+Vector search finds similar items based on the semantic meaning of the data, while keyword search relies on exact word matches or phrase occurrences. Vector search can handle unstructured, multimodal data (like text, images or audio) and find related concepts, while keyword search is more suited for structured text data where exact phrasing is important.
+
+### How is vector search conducted at scale?
+
+Vector databases use Approximate Nearest Neighbor (ANN) algorithms to speed up search time for large datasets. With ANN algorithms, vector search can return similar items to the query within a few milliseconds, even out of billions of objects.
+
+## Summary
+
+A quick recap:
+* Vector search, also called semantic search, can identify related objects without requiring exact text matches, allowing users to search based on semantic meaning rather than exact keywords.
+* It uses machine learning models to generate vector embeddings for all data objects and the query, and doing math calculations to determine similarity.
+* Vector embeddings capture the meaning and context of data.
+* Vector databases offer super fast responses to queries thanks to ANN algorithms.
+* ANN algorithms trade a small amount of accuracy for huge gains in speed.
+* Different types of vector search include hybrid search or multimodal search for images, audio, or video.
+* Use cases of vector search include Retrieval Augmented Generation (RAG), recommendation systems, or search systems.
+
+import WhatsNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2024-11-26-aws-partnership/img/hero.png b/blog/2024-11-26-aws-partnership/img/hero.png
new file mode 100644
index 0000000000..7e19980a36
Binary files /dev/null and b/blog/2024-11-26-aws-partnership/img/hero.png differ
diff --git a/blog/2024-11-26-aws-partnership/index.md b/blog/2024-11-26-aws-partnership/index.md
new file mode 100644
index 0000000000..361c6cdef0
--- /dev/null
+++ b/blog/2024-11-26-aws-partnership/index.md
@@ -0,0 +1,67 @@
+---
+title: "Weaviate on AWS: Our Journey as an AWS GenAI Startup"
+slug: aws-and-weaviate
+authors: [jobi, tony]
+date: 2024-11-26
+tags: ['partnerships']
+image: ./img/hero.png
+description: "Learn about the partnership between AWS and Weaviate."
+---
+
+
+
+With AWS [re:Invent 2024 approaching](https://reinvent.awsevents.com/), we would like to reflect on our remarkable journey with Amazon Web Services (AWS) and share how our partnership has evolved over the past eighteen months. What began as initial integration discussions has developed into a comprehensive collaboration that continues to provide value to our customers.
+
+From day one, we recognized that cloud hyperscalers would be critical to our go-to-market strategy. We strategically bet on AWS, seeing strong alignment in target market segments. While we compete with first-party cloud offerings like many ISVs, we've found that this healthy competition **drives innovation** and ultimately **benefits** and **gives choices to our customers**. Customers choose what is best for them, and this is a value that both the companies align on for the [ultimate customer centricity](https://aws.amazon.com/executive-insights/content/the-imperatives-of-customer-centric-innovation/).
+
+## Embracing the Co-build, Co-market, Co-sell model
+
+Establishing our partnerships on a solid foundation required us to invest upfront in key differentiators. We focused on taking the time to build these elements before scaling our go-to-market strategy. We aimed to invest wisely, create **strong differentiators**, and ensure our selling motions aligned with internal and external stakeholders. With this solid foundation, we are now fostering a win-win partnership and delighting our joint customers.
+
+## Building on Solid Ground
+
+The Weaviate core database runs natively on Amazon Elastic Kubernetes Service (EKS), providing enterprise-grade scalability and deployment flexibility. By leveraging Kubernetes, we can cater to our customers' needs, whether they require a serverless, multi-tenant solution managed by us or an enterprise-grade, single-tenant option. In the latter case, customers can use our managed services or deploy the same software directly from their environment, receiving enterprise support from Weaviate.
+
+We have expanded our technical integration portfolio to include native support for both Amazon SageMaker, aimed at data engineers building applications from the ground up, and Amazon Bedrock, which caters to data scientists who prefer a straightforward, no-fuss approach to deploying generative AI applications with their chosen foundation models. This expansion continues our commitment to supporting all user personas we target.
+
+Most of the Generative AI buzz is over GPUs and model training, but many production applications run fine on commodity hardware. With cost performance becoming a bigger and bigger issue for production-scale deployments, we are seeing a trend toward AWS Graviton-based deployments. Today, a handful of customers happily run their Knowledge Bases and Q&A apps running fully on gravitation SKUs.
+
+## Co-Marketing Success
+
+We have focused on telling a compelling "better together" story with AWS from the beginning. Our account and partner marketing teams at AWS have provided exceptional support for early-stage startups like ours. Additionally, with the executive sponsorship at the AWS leadership level, we receive the necessary support to grow our early go-to-market strategy. So far, our co-marketing efforts have produced impressive results over the past year:
+
+* Conducted four successful joint roadshows in San Francisco, Chicago, Seattle, and London.
+* Secured over 20+ customer wins with Morningstar, Instabase, Cisco, Bunq, and others
+* Established joint marketing campaigns, case studies, and webinars, generating significant results.
+* Utilized Marketing Development Funds (MDF) effectively, with a strategic focus on AWS MDF funds.
+* Implemented digital marketing initiatives, including co-branded content creation.
+* Developed success stories, case studies, and solution briefs.
+* Participated at re:Invent 2023 and in the partner showcase events like spotlight.
+
+We have also achieved significant program milestones, such as AWS Generative AI Competency certification, and we are a top-tier participant in the ISV Accelerate program. This success could not have happened without the contributions from teams throughout Weaviate, especially the outstanding work of our partner marketing team.
+
+## Meeting Customers where they are
+
+We have focused on close collaboration with AWS sellers from the beginning, opening up new joint opportunities. Tapping into cloud commitment drawdown through private offers has been a significant accelerator.
+
+Understanding that customers have varying needs and preferences, whether in deployment or procurement, we've structured our offerings to provide maximum flexibility. Whether through marketplace purchases, cloud commitment drawdowns, or direct engagement, we're committed to making Weaviate accessible through their preferred channels. Cloud marketplaces are essential to our customer journey. Whether customers are trying out SaaS serverless listings or testing Weaviate through its AWS Kubernetes marketplace listing, these marketplaces play a crucial role.
+
+## Join us at re:Invent
+
+As we prepare for AWS re:Invent in Las Vegas from December 2nd to December 5th, we can't wait to talk to you in person and hear about your generative AI journeys. We invite you to [visit us at re:invent](https://events.weaviate.io/aws-reinvent-2024) to learn more about how Weaviate and AWS work together to deliver the next generation AI-native stack.
+
+We'll demonstrate our latest integrations, share customer success stories, and provide insights into our roadmap for 2024. If you're attending re:Invent, we'd love to connect and discuss how Weaviate and AWS can help accelerate your AI initiatives.
+
+The past eighteen months have been transformative for our partnership with AWS, but we believe the best is yet to come. See you at re:Invent!
+
+What you can expect to see when you visit our booth, **#1506**:
+
+* A few demos showcasing hybrid search, Generative Feedback Loops (GFL), and Retrieval Augmented Generation (RAG)
+* Our new Weaviate comic book \- “The Vector Vault”
+* Limited edition stickers & hats
+* Weaviate's friendly experts
+
+We'll share a few demos showing the power of Weaviate and our features. We're expanding [Wealingo](https://github.com/weaviate/wealingo-demo) to include features like multi-tenancy, Generative Feedback Loops (GFLs), and more.
+
+We will have daily prizes given to the lucky winners of our raffle. You can also book a meeting with us in our private meeting space to learn all about Weaviate, our integration with AWS, and more. [Book your slot today](https://calendly.com/events-weaviate/weaviate-aws-reinvent), space is limited.
+
diff --git a/blog/2024-12-03-weaviate-embeddings/img/hero.jpg b/blog/2024-12-03-weaviate-embeddings/img/hero.jpg
new file mode 100644
index 0000000000..0aed51687c
Binary files /dev/null and b/blog/2024-12-03-weaviate-embeddings/img/hero.jpg differ
diff --git a/blog/2024-12-03-weaviate-embeddings/img/weaviate-embeddings.png b/blog/2024-12-03-weaviate-embeddings/img/weaviate-embeddings.png
new file mode 100644
index 0000000000..d75eb4a216
Binary files /dev/null and b/blog/2024-12-03-weaviate-embeddings/img/weaviate-embeddings.png differ
diff --git a/blog/2024-12-03-weaviate-embeddings/index.mdx b/blog/2024-12-03-weaviate-embeddings/index.mdx
new file mode 100644
index 0000000000..09e70656ed
--- /dev/null
+++ b/blog/2024-12-03-weaviate-embeddings/index.mdx
@@ -0,0 +1,53 @@
+---
+title: Introducing Weaviate Embeddings
+slug: introducing-weaviate-embeddings
+authors: [alvin, charles-pierse]
+date: 2024-12-03
+tags: ['release']
+image: ./img/hero.jpg
+description: "Say goodbye to the headaches of creating and managing vector embeddings."
+---
+
+
+
+If you’re building AI applications and have struggled with embedding provider rate limits, being locked in with a single provider or model, or the overhead of self-hosting models, you’re in good company. So many developers face unnecessary friction when it comes to creating and managing vector embeddings, a critical function of their AI pipelines.
+
+We’re excited to announce that [**Weaviate Embeddings**](/developers/wcs/embeddings), a new embedding service in Weaviate Cloud that makes creating vector embeddings as easy as possible for AI developers, is now available in preview.
+
+## The need for model flexibility
+
+The landscape of Gen AI and machine learning models is evolving at a crazy pace, with new models being released almost daily. From large language models (LLMs), to vertical-specific, multimodal, open-source, and commercial models—developers are faced with an overwhelming amount of options. When you’re locked in with a single provider, it’s difficult to swap models and try new ones. Open-source models are becoming increasingly popular, but the complexity of self-hosting is one many teams don’t have the time or inclination to do, they simply want to focus on building new application capabilities and creating new business outcomes.
+
+Weaviate has always prioritized model flexibility in our architecture, and we’ll continue to offer easy ways to integrate with external model providers via pre-built APIs. Weaviate Embeddings introduces the ability to run embedding models directly within Weaviate Cloud, without the need to connect to an external provider.
+
+In preview, Weaviate Embeddings offers Snowflake’s [Arctic-Embed](https://github.com/Snowflake-Labs/arctic-embed) open-source text embedding model which has scored impressively in industry benchmarks (although we’re fans of you running your own benchmarks). You can learn more about it [here](https://www.snowflake.com/engineering-blog/arctic-embed-m-v1-5-enterprise-retrieval/). We’ll soon add support for class-leading commercial multilingual and multimodal models. Users will be able to create vector embeddings for a variety of models, and compare the results and recall accuracy for their use case. This simplifies selecting the right model for the problem you are trying to solve.
+
+## Freedom from rate limits
+
+Most external embedding APIs impose strict rate limits or batch processing with lengthy turnaround times, which slows down your entire pipeline, or even worse creates unnecessary latency for the search experience you are building. One of our customers recently told us that they’ve had to rebuild their entire document syncing pipeline around the fact that they were constantly running into rate limits with their commercial provider. Other customers struggle with throughput, stability, cost, and recall quality using existing services.
+
+With Weaviate Embeddings, there are no artificial constraints. Our infrastructure is built to handle high-throughput use cases, efficiently processing large volumes of embeddings per second without compromising performance. Whether you’re importing billions of documents or performing real-time vector searches, Weaviate Embeddings scales with you, without slowing you down. Pricing is based on tokens consumed.
+
+
+
+
+## AI-ready data, with less friction
+
+Our goal is to give developers flexibility of choice, while providing levers to make building AI applications easier and more efficient. With Weaviate Embeddings, you get:
+
+* **Leading OSS and proprietary models:** Access cutting-edge open source and proprietary models. At preview, we will offer Snowflake Arctic-Embed (open-source) with commercial models coming early next year.
+* **Co-located models and data:** Reduce latency and improve performance by hosting both your data and your models in Weaviate Cloud.
+* **Cost efficient and GPU-powered:** Leverage GPU-powered inference and cost-efficient performance. Pay-as-you-go pricing based on tokens consumed means you only pay for what you use. You get full control, including the ability to optimize embedding dimensions based on your specific use case.
+* **Secure, enterprise-ready deployment:** Weaviate is SOC2-certified, with security features like role-based access controls and strict data isolation.
+
+## A foundation for the future
+
+Weaviate Embeddings is the bedrock of our future hosted model offerings. Upcoming advancements and releases will include multimodal models, dedicated hosting, and fine-tuned, optimized models tailored to your collection—laying the groundwork for exceptional retrieval quality across any domain. And this is just the beginning—stay tuned for exciting updates in the months ahead!
+
+## Give it a try
+
+To learn more about Weaviate Embeddings, check out the [docs](https://weaviate.io/developers/wcs/embeddings), sign up for our [upcoming webinar](https://webinars.techstronglearning.com/simplify-building-ai-native-embedding-models-and-vector-databases), or request [preview access](https://events.weaviate.io/embeddings-preview) and we’ll get in touch.
+
+import WhatsNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2024-12-04-unbody-weaviate-2/img/hero.jpg b/blog/2024-12-04-unbody-weaviate-2/img/hero.jpg
new file mode 100644
index 0000000000..5640f37fd3
Binary files /dev/null and b/blog/2024-12-04-unbody-weaviate-2/img/hero.jpg differ
diff --git a/blog/2024-12-04-unbody-weaviate-2/index.mdx b/blog/2024-12-04-unbody-weaviate-2/index.mdx
new file mode 100644
index 0000000000..5b15208a3d
--- /dev/null
+++ b/blog/2024-12-04-unbody-weaviate-2/index.mdx
@@ -0,0 +1,182 @@
+---
+title: 'Unbody’s Approach to Expanding Weaviate’s Generative Capabilities'
+slug: unbody-weaviate-extension
+description: "Unbody has enhanced Weaviate's generative capabilities by developing a custom Generative API that addresses limitations such as static project configurations, limited RAG syntax, and text-only input restrictions."
+authors: [hossein]
+date: 2024-12-04
+tags: []
+image: ./img/hero.jpg
+---
+
+
+
+Two years ago, we faced a critical question at [Unbody](https://unbody.io/): how could we build a development stack that was not just AI-enabled but truly AI-native? Like many developers, we started with familiar tools like MongoDB, but we quickly realized that traditional databases weren’t designed to handle the demands of AI-driven workflows. That’s when we turned to [Weaviate](https://weaviate.io/), an AI-native vector database.
+
+Unbody is an AI-native development stack designed to unify AI and product development into a single workflow. Weaviate fits perfectly into this vision, as [this blog post](/blog/unbody-weaviate) outlined. It became a core part of our stack, powering generative features and helping us deliver a flexible, AI-ready experience.
+
+Fast-forward to today: while Weaviate has been a strong foundation, we’ve encountered limitations in its generative capabilities as our needs evolved. Instead of moving away, we decided to expand it. By building a custom extension to its generative endpoint, we tailored it specifically for Unbody’s requirements.
+
+In this post, we’ll share how we extended Weaviate to create a more dynamic and adaptable generative API for our unique use case.
+
+## Challenge
+
+While Weaviate's [generative module (Retrieval Augmented Generation (RAG))](/developers/weaviate/configuration/modules#generative-model-integrations) initially met our user's needs, we encountered limitations as their requirements became more sophisticated. The [generative features](/developers/weaviate/search/generative) proved inflexible when dealing with evolving demands, especially for advanced scenarios.
+
+### 1. Static Configuration at Project Creation
+
+Weaviate's generative search module supported multiple generative providers, such as OpenAI, Cohere, PaLM, GPT4All, and others. However, at the project creation stage, users were required to lock in specific settings, including:
+
+- Model name
+- Temperature
+- Top-k, max tokens, and other related parameters
+
+While this setup worked well in the early stages, it presented significant challenges later. If a different generative provider or parameter was required, our users had to create a new project —a project in Unbody is an isolated data space— with the new settings. This meant deploying an entirely new Weaviate instance to accommodate the changes. This added friction when adapting to users' evolving needs and made the process inefficient.
+
+:::info
+This [feature was enabled](https://github.com/weaviate/weaviate/pull/6112) in Weaviate [`v.1.27.1`](https://github.com/weaviate/weaviate/releases/tag/v1.27.1)
+:::
+
+### 2. Limited Retrieval-Augmented Generation (RAG) Syntax
+
+Another major issue was the limited flexibility in Weaviate’s support for Retrieval-Augmented Generation (RAG). While simple prompts combined with data retrieval from the database worked in some cases, more complex interactions often required:
+
+- **Separation of System and User Prompts:** In scenarios where users interact with data, there’s often a need to distinctly separate the system’s instructions (system prompt) from the user’s input (user prompt). This becomes essential for use cases like chatbots or complex conversation flows. Without this, supporting advanced RAG functionalities or chat completions wasn’t feasible.
+- **Simplistic Template Engine:** The built-in template engine was relatively basic. It allowed users to insert string-typed fields directly from the objects they were executing RAG on. However, many users required a more powerful templating system that could handle advanced scenarios like dynamic data injection, context creation, or flexible template structures. There was no built-in way to filter or transform data, which limited our ability to create custom structures needed for more sophisticated workflows.
+
+### 3. Text-Only Input Limitation
+
+A third challenge arose when it came to input types. The generative method within Weaviate only accepted textual content as input, regardless of the model being used. This posed a limitation for our users' projects where they needed to use images as input. Being restricted to text input limited our users’ ability to fully leverage the capabilities of multi-modal models, preventing them from building more interactive and media-rich applications.
+
+## Solution
+
+The challenges we faced with Weaviate’s built-in generative features led us to decouple the generative module from the database to achieve increased flexibility.
+
+To address these needs, we developed a **Generative API** and created a custom module that integrates Weaviate into this API. This solution provides us with greater flexibility and scalability.
+
+This solution provides the ability to dynamically select models and providers, ensuring that Unbody projects are no longer tied to the static configurations set during project creation. This flexibility allows developers to switch models or adjust parameters on the fly, without having to redeploy or set up new projects.
+
+Additionally, the new generative module overcomes the limitations of text-only inputs, now supporting image-based prompts as well. This opens up multi-modal capabilities, enabling richer interactions and media-driven applications. We also introduced a more sophisticated templating engine that allows for advanced data manipulation and transformation, providing custom context creation and more complex workflows.
+
+With these improvements, our solution now offers:
+
+**Dynamic Model and Parameters**
+
+Users can switch between models like GPT-4 or Cohere without the need for redeployment, ensuring adaptability for a range of generative tasks. Additionally, it's possible to customize generative requests further with various parameters such as max tokens and temperature. This ensures more control over the generated content.
+
+```graphql
+{
+ Get {
+ TextBlock {
+ _additional {
+ generate(
+ groupedResult: {
+ task: """
+ prompt...
+ """
+ properties: ["text"]
+ options: {
+ model: "gpt-4o",
+ maxTokens: 1024,
+ temperature: 0
+ }
+ }
+ ) {
+ groupedResult
+ }
+ }
+ }
+ }
+}
+```
+
+**Multi-Modal Input Support**
+
+The module allows image inputs alongside text, unlocking the potential of models like GPT-4 Vision for richer, interactive applications.
+
+```graphql
+await unbody.get.imageBlock
+ .limit(2)
+ .generate.fromMany({
+ messages: [
+ {
+ content: 'Describe these images',
+ },
+ {
+ type: 'image',
+ content: '$..url',
+ },
+ ],
+ })
+ .exec()
+```
+
+**Enhanced Chat Completions**
+
+The system supports conversation flows by distinguishing between system and user roles, making it ideal for building interactive chatbots and retrieval-augmented generation (RAG) scenarios.
+
+```tsx
+await unbody.get.textBlock
+ .generate.fromMany({
+ messages: [
+ {
+ role: "system",
+ content: 'System prompt...',
+ },
+ ...history,
+ {
+ role: "user",
+ content: "user prompt"
+ }
+ ],
+ })
+ .exec()
+```
+
+**Advanced Data Transformation**
+
+Our Generative API integrates JQ, a powerful JSON processor, to enable sophisticated data transformation directly within API calls. This integration allows developers to dynamically filter, manipulate, and transform JSON data on-the-fly, eliminating the need for separate data processing steps. Using the `vars` option, query results can be transformed into JSON-serializable data and injected into message content, providing a streamlined workflow for complex data handling.
+
+```json
+vars: [
+ {
+ "formatter": "jq",
+ "name": "document",
+ "expression": "{ originalName, text }"
+ }
+]
+
+await unbody.get.googleDoc
+ .generate.fromOne({
+ messages: [
+ {
+ content: "prompt... \n DOCUMENT: {document}",
+ }
+ ],
+ })
+ .exec()
+```
+
+## What's Next
+
+As we continue to refine and expand our generative capabilities, we're focused on introducing several key features to enhance flexibility, performance, and functionality. Here's a glimpse of what's on our roadmap:
+
+### 1. Structured Output
+
+We plan to introduce structured output capabilities, allowing developers to enforce specific formats in model responses. By implementing output validation and schema support, we'll ensure that generated content follows predefined structures, making it seamless to integrate with existing systems and workflows.
+
+### 2. Support for Streaming
+
+To improve the responsiveness of our generative models, we are working on adding streaming support to both Unbody’s Generative API (REST) and the GraphQL API. This will enable users to receive partial outputs as they are generated, reducing latency and improving user experience, particularly for applications like real-time chat or large content generation tasks.
+
+### 3. Tool/Function Calling
+
+We’re exploring ways to integrate support for tool and function calling within the generative API. This feature would enable models to automatically trigger predefined functions or interact with external systems, enhancing their ability to handle complex workflows and dynamic tasks without manual intervention.
+
+### 4. Broader Model and Provider Support
+
+We plan to expand the range of supported providers and models, including fine-tuned models designed for specific tasks, such as coding. This will offer developers more options to tailor their generative models to specialized use cases, whether it’s code generation, legal document drafting, or other domain-specific applications.
+
+
+import WhatNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2024-12-11-hybrid-search-ja/img/hero.png b/blog/2024-12-11-hybrid-search-ja/img/hero.png
new file mode 100644
index 0000000000..10d80c5425
Binary files /dev/null and b/blog/2024-12-11-hybrid-search-ja/img/hero.png differ
diff --git a/blog/2024-12-11-hybrid-search-ja/img/hybrid-search-results.jpg b/blog/2024-12-11-hybrid-search-ja/img/hybrid-search-results.jpg
new file mode 100644
index 0000000000..b03b0fec8e
Binary files /dev/null and b/blog/2024-12-11-hybrid-search-ja/img/hybrid-search-results.jpg differ
diff --git a/blog/2024-12-11-hybrid-search-ja/img/hybrid-search.jpg b/blog/2024-12-11-hybrid-search-ja/img/hybrid-search.jpg
new file mode 100644
index 0000000000..c5f6178f6a
Binary files /dev/null and b/blog/2024-12-11-hybrid-search-ja/img/hybrid-search.jpg differ
diff --git a/blog/2024-12-11-hybrid-search-ja/img/keyword-search-results.jpg b/blog/2024-12-11-hybrid-search-ja/img/keyword-search-results.jpg
new file mode 100644
index 0000000000..3cd242f78c
Binary files /dev/null and b/blog/2024-12-11-hybrid-search-ja/img/keyword-search-results.jpg differ
diff --git a/blog/2024-12-11-hybrid-search-ja/index.mdx b/blog/2024-12-11-hybrid-search-ja/index.mdx
new file mode 100644
index 0000000000..5652f63369
--- /dev/null
+++ b/blog/2024-12-11-hybrid-search-ja/index.mdx
@@ -0,0 +1,229 @@
+---
+title: 'ハイブリッド検索とは？'
+slug: hybrid-search-ja
+description: "ハイブリッド検索の説明とKAGOME_JAトークナイザーを使った実装"
+authors: [leonie]
+date: 2024-12-11
+tags: ['concepts']
+image: ./img/hero.png
+---
+
+
+
+ハイブリッド検索は、複数の検索アルゴリズムを組み合わせることで、検索結果の精度や関連性を向上させる技術です。一般的に、ハイブリッド検索とはキーワード検索とベクトル検索を組み合わせた方法を指します。これら二つのアルゴリズムの強みを活用することで、ユーザーに効果的な検索体験を提供します。
+
+Weaviate[ベクトルデータベース](/blog/what-is-a-vector-database)におけるハイブリッド検索では、スパースベクトルとデンスベクトルの両方が利用されます。
+スパースベクトルにはトークナイザーが必要ですが、Weaviate `v1.28` では日本語テキスト向けに新しいトークナイザー`KAGOME_JA`が導入されました。
+
+このブログ記事では、ハイブリッド検索の基本を学び、日本語テキストで`KAGOME_JA`トークナイザーを使ったWeaviateのハイブリッド検索の方法を解説します。
+
+## ハイブリッド検索の説明
+
+Weaviateのハイブリッド検索は、デンスベクトルとスパースベクトルを統合して、それぞれの検索手法の利点を活かします。
+これら二つのベクトルは異なるアルゴリズムによって計算されます。
+
+
+
+### キーワード検索
+
+キーワード検索にはスパースベクトルを使用します。
+スパースベクトルは、主にゼロ値が多く一部に非ゼロ値を持つ構造をしており、デンスベクトルは主に非ゼロ値で構成されています。
+スパース埋め込みは、[BM25](https://en.wikipedia.org/wiki/Okapi_BM25) や [SPLADE](https://arxiv.org/abs/2107.05720) のようなアルゴリズムから生成されます。
+Weaviateにおける現在のハイブリッド検索実装では、BM25/BM25Fが使用されています。
+
+スパース埋め込みを生成するためには、トークナイザーが必要です。
+Weaviateでは、日本語テキスト向けに以下の[3つのトークナイザーが利用可能](/developers/academy/py/tokenization/options)です。
+
+| トークナイザー | `TRIGAM` | `GSE` | `KAGOME_JA` （Weaviate `v1.28`で導入） |
+| :---- | :---- | :---- | :---- |
+| メリット | 辞書不要 部分一致する | 辞書にある単語をできるだけ 部分一致する | 辞書にある単語 不要な結果が出ない |
+| デメリット | 不自然な単語がある 出てくる単語が多数 | 英単語は1文字ずつに分割 不自然な単語がある 辞書が必要 | 単語中の部分一致はしない 辞書が必要
+*Source: [Jun Ohtani](https://x.com/johtani)*
+
+### ベクトル検索
+
+ベクトル検索にはデンスベクトルを使用します。
+デンス埋め込みは、[GloVe](https://text2vec.org/glove.html)や[Transformers](https://huggingface.co/docs/transformers/index)といった機械学習モデルから生成されます。これらのベクトルは情報が密に詰まっており、ほとんどが非ゼロ値で構成されています。ベクトルデータベースはこれらの埋め込みを保存し、二つのベクトル間の距離を計算します。この[距離メトリクス](/blog/distance-metrics-in-vector-search)により、二つのベクトル埋め込みがどれほど類似しているか、または異なるかがわかります。検索クエリもデータベクトルと同様にベクトルに変換され、その距離値によってベクトルの近さが決まります。
+
+### 融合アルゴリズム
+
+キーワード検索とベクトル検索の結果を単一のランキングリストに統合する方法はいくつかあります。Weaviate `v1.20`以降では、以下の2つのアルゴリズムが利用可能です：
+
+* `rankedFusion`
+* `relativeScoreFusion`
+
+これらの融合アルゴリズムについての詳細は、[こちらの記事](/blog/hybrid-search-fusion-algorithms#fusion-algorithms)で解説されています。
+
+さらに、alphaパラメータを用いてキーワード検索とベクトル検索の結果の重み付けを調整できます：
+
+* `alpha = 0`：キーワード検索のみ
+* `alpha = 1`：ベクトル検索のみ
+* `alpha = 0.5`：スパースとデンスのベクトルを均等に重み付け
+
+## ハイブリット検索のメリット
+
+ハイブリッド検索は、ベクトル検索の機能を活用しつつ、正確なキーワード一致も必要とする検索システムに理想的です。
+
+例えば、「春のドレス」というクエリを考えてみましょう。通常のキーワード検索やベクトル検索に比べて、ハイブリッド検索を使用する方がより良い結果を得られます。デンスベクトル表現では「春」が花柄や軽い素材を意味すると解釈され、キーワード検索では「ドレス」という単語に正確に一致します。
+このように、ハイブリッド検索はスパースベクトルとデンスベクトルの両方の利点を活かします。
+
+一般的に、デンスベクトルはクエリの文脈を理解するのに優れており、スパースベクトルはキーワード一致に優れています。
+
+## Weaviateのハイブリッド検索 実装
+
+これで、ハイブリッド検索の概要と日本語テキストへの応用がわかるようになったと思います。`KAGOME_JA`トークナイザーを活用して、Weaviateでさらに効果的な検索を実現しましょう！
+
+```py
+data = [
+ ["スポーツ用レギンス", "吸湿速乾性に優れた素材で作られたレギンス。ジムやヨガ、ランニングに最適です。"],
+ ["点線模様の長靴", "雨の日のおしゃれにぴったりの長靴。ユニークな点線模様が特徴です。"],
+ ["トレンチコート", "クラシックなデザインのトレンチコート。春や秋の軽い羽織りに最適な一着です。"],
+ ["ひまわりの模様のドレス", "明るいひまわり柄が夏らしい印象のドレス。軽やかで涼しい素材が魅力です。"],
+ ["ライトカーディガン", "肌寒い日の重ね着に便利なライトカーディガン。軽量で持ち運びにも便利です。"],
+ ["フローラルミニスカート", "春や夏にぴったりの花柄ミニスカート。華やかなデザインで女性らしさを引き立てます。"],
+ ["ピンクのスカート", "柔らかなピンク色がかわいらしいスカート。シンプルなデザインでコーディネートしやすいです。"],
+ ["カシミアセーター", "高級感のあるカシミア素材で作られたセーター。寒い冬に暖かさとスタイルを提供します。"],
+ ["ライトウールスカーフ", "秋や冬にぴったりのウールスカーフ。軽量で肌触りが良く、暖かさをプラスします。"],
+ ["ニットドレス", "柔らかく暖かい素材で作られたニットドレス。寒い季節のカジュアルスタイルに最適です。"],
+ ["ミディプリーツスカート", "エレガントなシルエットが特徴のプリーツスカート。どんなトップスとも相性抜群です。"],
+ ["花柄の傘", "雨の日を明るくしてくれる花柄デザインの傘。軽量で使いやすい構造です。"],
+ ["長袖の紺色のドレス", "落ち着いた紺色がエレガントな長袖ドレス。フォーマルな場にも最適です。"],
+ ["春の花柄ドレス", "春の装いにぴったりの鮮やかな花柄ドレス。柔らかい素材で着心地も抜群です。"],
+ ["ボア付きデニムジャケット", "冬の寒さから守るボア付きデニムジャケット。カジュアルスタイルにぴったりです。"],
+ ["レザーアンクルブーツ", "シックで洗練されたデザインのレザーアンクルブーツ。秋冬の装いにおすすめです。"],
+ ["防水ジャケット", "雨の日の外出を快適にする防水ジャケット。軽量で動きやすいデザインが魅力です。"],
+ ["シンプルな黒のタートルネック", "どんなスタイルにも合わせやすい黒のタートルネック。秋冬の重ね着に最適です。"],
+ ["シフォンブラウス", "軽やかで女性らしいデザインのシフォンブラウス。春や夏のフォーマルな場にもおすすめです。"]
+]
+```
+
+### 1. Weaviate データベース作成
+
+日本語のKAGOMEトークナイザーを使用したハイブリッド検索をWeaviateで利用するには、Weaviate`v1.28`以降のバージョンを使用していることを確認する必要があります。以下の手順で、[Dockerを使用してWeaviateベクトルデータベースのインスタンスをローカル環境にセットアップ](/developers/weaviate/installation/docker-compose)できます：
+
+```
+docker run -p 8080:8080 -p 50051:50051 -e ENABLE_TOKENIZER_KAGOME_JA='true' -e ENABLE_MODULES='text2vec-cohere' cr.weaviate.io/semitechnologies/weaviate:1.28.1
+```
+
+次は[Weaviate ClientのPythonライブラリ](/developers/weaviate/client-libraries)を準備します。バージョン4.9.6以上が必要です。
+
+```
+!pip install weaviate-client
+```
+
+### 2. Weaviateの接続情報をセットアップ
+
+次に、Weaviate Clientを初期化します。Weaviateに接続するための設定では、Cohereの埋め込みモデルを使用するため、CohereのAPIキーが必要です。
+
+```py
+import weaviate, os
+
+# Connect to your local Weaviate instance deployed with Docker
+client = weaviate.connect_to_local(
+ headers={
+ "X-COHERE-Api-Key": os.environ["COHERE_API_KEY"]
+ }
+)
+```
+
+### 3. データコレクションの定義
+
+次に、クラスを作成して[Schema](developers/weaviate/config-refs/schema)を定義します。クラスは、オブジェクトを格納するデータコレクションです。クラスのSchemaには、Cohereの多言語（日本語対応）埋め込みモデル `embed-multilingual-v3.0` を定義します。また、`KAGOME_JA`トークナイザーを有効化します。
+
+```py
+import weaviate.classes.config as wc
+
+client.collections.create(
+ name="ClothingItems",
+ vectorizer_config=wc.Configure.Vectorizer.text2vec_cohere(
+ model="embed-multilingual-v3.0", ),
+ properties=[
+ wc.Property(name="item",
+ data_type=wc.DataType.TEXT,
+ tokenization=wc.Tokenization.KAGOME_JA
+ ),
+ wc.Property(name="description",
+ data_type=wc.DataType.TEXT,
+ tokenization=wc.Tokenization.KAGOME_JA
+ ),
+ ]
+)
+```
+
+### 4. データ登録
+
+次に、WeaviateのClientを使用してデータを登録します。
+
+```py
+# Get a collection object for "ClothingItems"
+clothes = client.collections.get("ClothingItems")
+
+with clothes.batch.dynamic() as batch:
+ for d in data:
+ batch.add_object({
+ "item": d[0],
+ "description" : d[1],
+ })
+```
+
+### 5. クエリ方法
+
+これでベクトルデータベースのセットアップが完了し、クエリを実行できる状態になりました。
+
+まずは、シンプルなBM25クエリから始めましょう。
+
+```py
+response = clothes.query.bm25(
+ query="ドレス",
+ limit=3
+)
+
+for item in response.objects:
+ print(f"{item.properties['item']}: {item.properties['description']}")
+```
+
+結果は以下の通りです。ご覧のとおり、検索結果には「ドレス」というキーワードに一致するアイテムのみが含まれています。
+
+```
+長袖の紺色のドレス: 落ち着いた紺色がエレガントな長袖ドレス。フォーマルな場にも最適です。
+ニットドレス: 柔らかく暖かい素材で作られたニットドレス。寒い季節のカジュアルスタイルに最適です。
+ひまわりの模様のドレス: 明るいひまわり柄が夏らしい印象のドレス。軽やかで涼しい素材が魅力です。
+```
+
+
+
+では、前の例で説明したハイブリッド検索クエリを試してみましょう。
+
+```py
+response = clothes.query.hybrid(
+ query="春のドレス",
+ alpha=0.8,
+ limit=3
+)
+
+for item in response.objects:
+ print(f"{item.properties['item']}: {item.properties['description']}")
+```
+
+結果は以下の通りです。ご覧のとおり、検索結果には「ドレス」というキーワードに一致するアイテムと、春に適した花柄のアイテムの両方が含まれています。最後のアイテムは「ドレス」というキーワードには一致していませんが、それでも花柄のスカートが含まれています。これは、alpha値を0.8に設定しており、キーワード検索よりもベクトル検索を優先しているためです。
+
+```
+春の花柄ドレス: 春の装いにぴったりの鮮やかな花柄ドレス。柔らかい素材で着心地も抜群です。
+ひまわりの模様のドレス: 明るいひまわり柄が夏らしい印象のドレス。軽やかで涼しい素材が魅力です。
+フローラルミニスカート: 春や夏にぴったりの花柄ミニスカート。華やかなデザインで女性らしさを引き立てます。
+```
+
+
+
+
+## まとめ
+
+この記事では、ハイブリッド検索の概念について説明しました。Weaviateベクトルデータベースにおけるハイブリッド検索は、キーワード検索とベクトル検索を組み合わせたものです。これら二つの検索を統合することで、ベクトル検索の強力な機能を活用しながら、製品名やブランド名など特定のキーワードの正確な一致も必要とするユースケースで、より関連性の高い結果を得ることができます。
+
+日本語テキストに対して、Weaviateではキーワード検索およびハイブリッド検索のために3つの異なるトークナイザーを選択できるようになりました。上記のKAGOME\_JAトークナイザーを使用したハイブリッド検索の例について、コードは[このノートブック](https://github.com/weaviate/recipes/blob/main/weaviate-features/hybrid-search/hybrid_search_cohere_japanese.ipynb)で確認できます。
+
+ハイブリッド検索の詳細については、[ドキュメント](/developers/weaviate/api/graphql/search-operators#hybrid)をご覧ください！
+
+import WhatNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2024-12-12-weaviate-1-28-release/_core-1-28-include.mdx b/blog/2024-12-12-weaviate-1-28-release/_core-1-28-include.mdx
new file mode 100644
index 0000000000..0e4ea82d48
--- /dev/null
+++ b/blog/2024-12-12-weaviate-1-28-release/_core-1-28-include.mdx
@@ -0,0 +1,147 @@
+Weaviate `1.28` brings enterprise-grade security, faster indexing, and improved multilingual support to your vector database. This release introduces role-based access control (RBAC), enhanced async indexing improvements, and native Japanese language support - making Weaviate more secure, faster, and globally accessible than ever.
+
+Here are the key highlights:
+
+Here are the release ⭐️*highlights*⭐️!
+
+
+
+- [**Improved security 👤🔐: Role-based access control preview**](#role-based-access-control-rbac️)
+- [**Speedier, more robust 🏎️ async vector indexing**](#async-indexing-improvements)
+- [**Conflict resolution improvements 🤝 for deletions**](#conflict-resolution-improvements)
+- [**New Japanese 🇯🇵 `kagome_ja` tokenizer for keyword/hybrid search**](#new-japanese-tokenizer)
+- [**Groundwork for Keyword & Hybrid search 🔠 improvements: BlockMax WAND**](#blockmax-wand-experimental)
+- [**Improvements since 1.27**](#other-improvements)
+ - [**Voyage AI Multimodal model support**](#voyage-ai-multimodal-model-support)
+ - [**Weaviate Embeddings**](#weaviate-embeddings)
+
+## Role-based access control (RBAC)️
+
+Weaviate `1.28` introduces role-based access control (RBAC) as a technical preview, offering more granular control over user permissions.
+
+:::caution 🚧 Technical Preview
+Role-based access control (RBAC) is added `v1.28` as a **technical preview**. This means that the feature is still under development and may change in future releases, including potential breaking changes. **We do not recommend using this feature in production environments at this time.**
+
+
+We appreciate [your feedback](https://forum.weaviate.io/) on this feature.
+:::
+
+This powerful new feature is another big step that is built for our power and enterprise users. It allows you to provide granular sets of permissions through built-in and custom roles to control access to your Weaviate instance.
+
+With RBAC, you can:
+
+- Assign built-in roles (`admin`, `viewer`) to users for common access patterns
+- Create custom roles with fine-grained permissions for different resources
+- Assign multiple roles to users
+- Control access at various levels including collections, objects, backups, and cluster operations
+
+
+
+For example, you could create roles like:
+
+A "data scientist" role with read-only access to specific collections
+An "app service" role that can only write to certain collections
+A "backup operator" role that can only manage backups
+A "devops" role with access to view cluster and node metadata
+
+Permissions can be as specific as needed - from broad access to all collections down to operations on individual collections. The system is additive, meaning if a user has multiple roles, they'll gain the combined permissions of all their roles.
+
+To learn more, check out our [Authentication](/developers/weaviate/configuration/authentication) and [Authorization and RBAC](/developers/weaviate/configuration/authorization) documentation, along with the [Roles](/developers/weaviate/configuration/roles) guide.
+
+While RBAC is currently in technical preview (meaning there might be changes before the final release), we're excited to get feedback from the community on this important security feature. For production environments, we recommend continuing to use the existing authentication and authorization mechanisms for now.
+
+## Async indexing improvements
+
+We've enhanced our [asynchronous vector indexing](/developers/weaviate/concepts/vector-index#asynchronous-indexing) feature in `v1.28`, making it more robust and capable than ever. While async indexing (introduced in `v1.22`) already helped speed up batch imports by handling vector indexing in the background, we've now expanded its capabilities and rebuilt its foundations.
+
+The async indexing system now handles all operations related to the vector index, including:
+- Single object imports
+- Object deletions
+- Object updates
+- All batch operations (as before)
+
+This results in a unified system that is more robust, and also more performant. One reason is that it reduces lock contention, which can slow down indexing operations if multiple requests are trying to update the index at the same time.
+
+Another change is that we've replaced the previous in-memory queue with an on-disk queue system. This architectural change reduces memory usage.
+
+This feature is particularly valuable when you're working with large datasets. While the object store updates immediately, vector indexing happens in the background, allowing your requests to complete faster.
+
+Want to try it out? Just set `ASYNC_INDEXING=true` in your environment variables. Weaviate Cloud users can enable it with a single click in the Cloud Console.
+
+As you can see, we are still making improvements to the async indexing feature. This is still marked as **experimental**, so we'd love to hear your feedback on how it's working for you.
+
+## Conflict resolution improvements
+
+When running Weaviate across multiple replicas, it's crucial to maintain [consistency](/developers/weaviate/concepts/replication-architecture/consistency) across the cluster. One common challenge is how to handle conflicts when objects are deleted, as a deleted object might be mistaken for an object that is yet to be created.
+
+`v1.28` introduces three deletion resolution strategies that you can choose from based on your needs:
+
+- `DeleteOnConflict`: The most conservative option that ensures deletions are always honored. Once an object is marked for deletion, it stays deleted across all replicas.
+- `TimeBasedResolution`: A timestamp-based approach where the most recent operation wins. For example, if an object is deleted and then recreated later, the recreation takes precedence. But if the deletion happened after the recreation attempt, the object stays deleted.
+- `NoAutomatedResolution`: The classic approach (and default setting) for those who prefer manual control over conflict resolution.
+
+These strategies work seamlessly with Weaviate's existing replication and repair features to ensure your data stays consistent across your cluster, while giving you control over how deletion conflicts should be handled.
+
+This improvement is particularly valuable for production deployments where data consistency is critical and manual intervention should be minimized.
+
+To use this feature, [set the deletion strategy at the collection level under replication settings](/developers/weaviate/manage-data/collections#replication-settings).
+
+### New Japanese tokenizer
+
+We 🫶 our community, and seeing Weaviate usage grow around the world. So we are very excited to show off this feature that expands our existing set of keyword tokenizers for Japanese text. Especially so as it was contributed to us by [Jun Ohtani](https://github.com/johtani), a Weaviate community user!
+
+This feature adds a new Japanese tokenizer, `kagome_ja`, to Weaviate's keyword and hybrid search capabilities. This tokenizer uses the [Kagome](https://github.com/ikawaha/kagome) library, a morphological analyzer for Japanese text, to break down Japanese text into individual tokens for indexing and searching.
+
+To use this tokenizer, set the `ENABLE_TOKENIZER_KAGOME_JA` [environment variable](/developers/weaviate/config-refs/env-vars) to `true`, and set the [tokenization method](/developers/weaviate/config-refs/schema#kagome_ja-tokenization-method) for the relevant properties to `kagome_ja`.
+
+## BlockMax WAND (Experimental)
+
+:::caution 🚧 Experimental Feature
+This iteration of BlockMax WAND is not for production use.
+
+
+Currently, this feature must be enabled before ingesting any data, as it requires a different form of an inverted index. For this experimental version, we do not support migrations. The disk format may change in future releases, which will require a re-index, meaning a potential re-ingestion of data.
+:::
+
+BM25 searches form the foundation of Weaviate's keyword and hybrid search capabilities, so we're always looking for further improvements.
+
+`v1.28` lays the groundwork for this by introducing an experimental implementation for the [BlockMax WAND](http://engineering.nyu.edu/~suel/papers/bmw.pdf) algorithm. This highly experimental feature aims to speed up BM25 and hybrid searches by more efficiently evaluating search results, using "blocks" of documents to reduce the number of calculations needed.
+
+While this feature isn't ready for production use, we're including it in 1.28 for those interested in testing and providing feedback. You can try it out by enabling two environment variables:
+
+- `USE_INVERTED_SEARCHABLE: true`: Enables an experimental disk format optimized for BlockMax WAND
+- `USE_BLOCKMAX_WAND: true`: Activates the BlockMax WAND algorithm for queries
+
+We're excited about the potential performance improvements this could bring to keyword and hybrid search in future releases.
+
+## Other improvements
+
+### Voyage AI Multimodal model support
+
+
+
+Weaviate recently added support for [Voyage AI's Multimodal model](https://blog.voyageai.com/2024/11/12/voyage-multimodal-3/), a powerful model that combines text and image embeddings for multimodal search.
+
+This model integration was added to Weaviate recently, with `v1.27.8`, `v1.26.12` and `v1.25.28`. You can now use the [Voyage AI model integration](/developers/weaviate/model-providers/voyageai) to build AI-driven applications that can search for objects based on both text and image data.
+
+Note: Client support and official documentation will follow very soon
+
+### Weaviate Embeddings
+
+
+
+In case you missed it, we recently introduced a new service called [Weaviate Embeddings](/developers/wcs/embeddings). This service provides secure, scalable embedding generation as a fully managed service, tightly integrated with Weaviate Cloud instances.
+
+This allows you to generate text embeddings using single authentication, and benefits of unified billing with your Weaviate Cloud account. Read more about it in our [announcement blog post](/blog/introducing-weaviate-embeddings).
+
+## Summary
+
+Ready to Get Started?
+
+Enjoy the new features and improvements in Weaviate `1.28`. The release is available open-source as always [on GitHub](https://github.com/weaviate/weaviate/releases/tag/v1.28.0), and will be available for new Sandboxes on [Weaviate Cloud](https://console.weaviate.cloud/) very shortly.
+
+For those of you upgrading a self-hosted version, please check the [migration guide](/developers/weaviate/more-resources/migration#general-upgrade-instructions) for detailed instructions.
+
+It will be available for Serverless clusters on Weaviate Cloud soon as well.
+
+Thanks for reading, see you next time 👋!
diff --git a/blog/2024-12-12-weaviate-1-28-release/img/authz-diagram.png b/blog/2024-12-12-weaviate-1-28-release/img/authz-diagram.png
new file mode 100644
index 0000000000..234db9b706
Binary files /dev/null and b/blog/2024-12-12-weaviate-1-28-release/img/authz-diagram.png differ
diff --git a/blog/2024-12-12-weaviate-1-28-release/img/hero.png b/blog/2024-12-12-weaviate-1-28-release/img/hero.png
new file mode 100644
index 0000000000..f10d7efcb1
Binary files /dev/null and b/blog/2024-12-12-weaviate-1-28-release/img/hero.png differ
diff --git a/blog/2024-12-12-weaviate-1-28-release/index.mdx b/blog/2024-12-12-weaviate-1-28-release/index.mdx
new file mode 100644
index 0000000000..c5cc7eb79c
--- /dev/null
+++ b/blog/2024-12-12-weaviate-1-28-release/index.mdx
@@ -0,0 +1,18 @@
+---
+title: Weaviate 1.28 Release
+slug: weaviate-1-28-release
+authors: [jp]
+date: 2024-12-12
+image: ./img/hero.png
+tags: ['release', 'engineering']
+description: "Read about a preview of role-based access control, improved indexing, Weaviate Embeddings, new Voyage Multimodal model and a Japanese BM25 tokenizer!"
+
+---
+
+import Core128 from './_core-1-28-include.mdx';
+
+
+
+import WhatsNext from '/_includes/what-next.mdx'
+
+
diff --git a/blog/2024-12-17-hardening-oss/img/harden.jpg b/blog/2024-12-17-hardening-oss/img/harden.jpg
new file mode 100644
index 0000000000..92b8bfa079
Binary files /dev/null and b/blog/2024-12-17-hardening-oss/img/harden.jpg differ
diff --git a/blog/2024-12-17-hardening-oss/index.mdx b/blog/2024-12-17-hardening-oss/index.mdx
new file mode 100644
index 0000000000..90435f6e33
--- /dev/null
+++ b/blog/2024-12-17-hardening-oss/index.mdx
@@ -0,0 +1,41 @@
+---
+title: 'Hardening your Weaviate OSS Installation'
+slug: hardening-oss
+description: "Moving your Weaviate OSS instance into production? Check out our handy guide to securing your Weaviate database in the cloud, including all the helpful modules we’ve added to make it as easy as possible to keep your vectors safe."
+authors: [spiros]
+date: 2024-12-17
+tags: [how-to]
+image: ./img/harden.jpg
+---
+
+
+
+At Weaviate, we take the protection of your data seriously. Our information security measures continuously evolve and adapt to threats we see out there in the wilds of the internet.
+An area we are seeing more and more within our community is successful Weaviate projects that have been put into production using our [Open Source product](https://github.com/weaviate/weaviate). With Weaviate Open Source you’re in control of your data, the Weaviate environment and the infrastructure it’s hosted on. In fact, we’ve built Weaviate with a range of [modules](/developers/weaviate/configuration/modules) that can make your transition from hackathon to startup to customer-ready easy and safe.
+Managing a Weaviate installation as part of a wider application at scale can be challenging and will introduce risks that are not faced by small projects or experiments. That’s why today we’re launching our [OSS Hardening Guide.](/img/site/Security-Checklist.pdf) This document contains hints and tips to use the built-in tools that are part of Weaviate and its hosting infrastructure in order to maximise the confidentiality, integrity and availability of your installation.
+
+## Risk-based thinking
+Not every Weaviate installation needs the same security investment as ‘Fort Knox’ but it is difficult to know how much security is appropriate when deploying in production.
+Behind the scenes in the Weaviate security team, we strongly believe that the best security and investment decisions are reached through risk-based thinking. The first step to hardening your Weaviate OSS installation should be to consider the risks that you face; is the data that you’ll be storing sensitive in any way? are you subject to any industry or national regulation? what about your end users - will they expect a certain level of security?
+Once you have determined your risks, it’s time to see what mitigations you can implement in order to manage them. Head to our OSS Hardening Guide to understand what is available within Weaviate. While measures such as [Role-based Access Control](/developers/weaviate/configuration/authorization), automatic [Backup](/developers/weaviate/configuration/backups) capability and [Replication](/developers/weaviate/configuration/replication) can help you design a set of easy-to-implement controls to manage access and availability, you will also want to consider encryption both for data in transit and in storage.
+Remember the principal of defense-in-depth: always configure multiple security controls in case one fails.
+
+:::info
+All modern information security frameworks require some form of risk assessment, so you're already ahead once your customers start asking for evidence of that practice!
+:::
+
+## Testing before production
+It’s a good idea to test your Weaviate security measures to make sure that they are correctly configured before you launch. In particular, we recommend that you use an online SSL checker to make sure your encryption in transit is working, and that you test that you can restore your backups should the worst happen.
+Now is a good time to go back to your risks and mitigations, and assure yourself that the measures you’re implemented are actually managing the risk - for instance, if you’ve used replication to make sure your service can survive if a node fails, have you tested to verify that’s the case?
+If this service is especially critical, you may wish to engage the service of a third party penetration testing company to assist you with your assurance. If you choose to do this, we recommend ensuring your tester is CREST certified.
+
+## Ongoing Maintenance
+Sadly, information security is not a ‘set and forget kind’ of job. There are a number of regular activities that we recommend are scheduled and operated as often as you think necessary. Remember that risk-based thinking should be practiced whenever a major change takes place in your environment, whether that’s the rollout of a new major version, the introduction of a new feature, expanding into a different customer base or reaching a company growth milestone. Involving a diverse range of thinkers can improve your identification, understanding and treatment of risk as well.
+We release new versions of Weaviate all the time, and strongly advise OSS users to stay within one major release of our latest version if possible. These releases will contain security fixes and improvements, as well as exciting new features. We will also patch any third party vulnerabilities that are identified by the open source community and our security partners.
+
+## Stay safe out there!
+In summary, going from development to production is an incredible journey, and our OSS Hardening Guide, your risk-based thinking coupled with some testing, it can be a safe and secure one as well.
+
+import WhatNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2024-12-18-recap-london-roadshow/img/hero.png b/blog/2024-12-18-recap-london-roadshow/img/hero.png
new file mode 100644
index 0000000000..d9d3d36866
Binary files /dev/null and b/blog/2024-12-18-recap-london-roadshow/img/hero.png differ
diff --git a/blog/2024-12-18-recap-london-roadshow/index.mdx b/blog/2024-12-18-recap-london-roadshow/index.mdx
new file mode 100644
index 0000000000..bdfb83acf0
--- /dev/null
+++ b/blog/2024-12-18-recap-london-roadshow/index.mdx
@@ -0,0 +1,115 @@
+---
+title: 'Recap: London Roadshow'
+slug: recap-london-roadshow
+description: "The Weaviate team was in London last month, where the presentations were packed with insights from the field, real-world use cases, and inspiration for what’s next in AI. From streamlining development to tackling challenges in nuclear energy, here’s a recap of the standout moments."
+authors: [staci]
+date: 2024-12-18
+tags: []
+image: ./img/hero.png
+---
+
+
+
+---
+
+**AI in Action: Lessons from the Real World**
+
+The Weaviate team was in London last month, where the presentations were packed with insights from the field, real-world use cases, and inspiration for what’s next in AI. From streamlining development to tackling challenges in nuclear energy, here’s a recap of the standout moments.
+
+---
+
+### **1. Mastering Vector Databases**
+
+Our CTO & Co-Founder, Etienne Dilocker dove deep into two emerging use cases for large-scale projects:
+
+- **E-commerce search** handling **1-10 billion embeddings** with sub-second queries, powered by HNSW indexing and smart optimizations.
+- **Email RAG** using a multi-tenant setup to process hundreds of thousands of smaller datasets with sub-second speed.
+
+Both examples proved there’s no one-size-fits-all solution—your approach should depend on the data and the problem you’re solving.
+
+---
+
+### **2. OLIVER’s AI Sandbox and Experimentation**
+
+Andy Hoare, Senior Business Analyst, from the marketing agency **OLIVER** shared how the company scaled from 500 to 5,000 people in 10 years, while making AI experimentation a core part of their strategy. Their **AI Sandbox** has **1,250+ users**, generating **10,000 images a month** and powering **10 million chat interactions**.
+
+He spoke about the following projects:
+
+- **Slipstream** is changing how clients and agencies work together by empowering clients to create more comprehensive and efficient briefs and allowing agencies to work smarter and faster.
+- **C2PA** is a transparency initiative for AI-generated content that enables OLIVER to maintain a duty of care to its clients and audiences and build trust as a content creator.
+- **Share of Model,** with Jellyfish, measures brand perception within AI models.
+
+Andy’s advice? Experiment often, fail fast, and find great tech partners to support your growth.
+
+
+
+
+
+---
+
+### **3. AI Meets Nuclear Decommissioning**
+
+Willem van Asperen and Derek Van Gelderen from **PA Consulting** shared their innovative use of AI in the nuclear sector. At **Sellafield Limited**, they’re solving business-critical challenges, like:
+
+- A wave of retirements shrinking their workforce by **20%** in the next five years.
+- Document reviews that used to take **12 months**.
+
+Their solutions included:
+
+- **SLComply.ai**, which automates compliance and slashes administrative time by **90%**.
+- **DANI**, a GenAI-powered assistant that processes documents in minutes instead of months.
+- **Genie**, a system that connects LLMs, databases, and operational tools to make workflows seamless.
+
+PA Consulting’s use of Weaviate to scale its document and compliance tools highlights the power of vector search and AI in transforming workflows. By integrating Weaviate, they’re achieving greater efficiency, accuracy, and scalability all while remaining user centric.
+
+
+
+
+
+---
+
+### **4. Building AI-Native Software with Unbody.io**
+
+Amir Huieh, CEO & Co-Founder of **unbody.io**, shared his journey from running a computer vision and NLP to founding Unbody, an AI-first startup. He broke down the difference between **AI-native** apps (where AI is the core) and **AI-enhanced** ones (where AI is just a bonus).
+
+Amir showed how integrating **Weaviate** helped his team streamline their development stack, cutting out unnecessary layers. Demos included:
+
+- A **healthcare platform** that matches patients with therapists.
+- An **AI blogging tool** that turns videos into full articles.
+- A **desktop organizer** for simplifying digital clutter.
+
+And the big news? **Unbody is going open source!** Amir invited the community to [jump in and contribute](http://unbody.io/blog/going-oss).
+
+
+
+
+
+---
+
+### **Key Takeaways**
+
+Across every session, a few key themes stood out:
+
+- **Simplify where you can**: Merging AI and traditional stacks saves time and headaches.
+- **Experiment boldly**: Whether it’s a sandbox or open source, trying new things (and failing fast) is the way to grow.
+- **AI is solving real problems**: From nuclear compliance to blogging tools, the impact is tangible and exciting.
+
+The future of AI is collaborative, creative, and just getting started. Let’s keep building, learning, and sharing!
+
+import WhatNext from '/_includes/what-next.mdx';
+
+
diff --git a/blog/2025-01-07-inngest-ai-workflows/img/demo.gif b/blog/2025-01-07-inngest-ai-workflows/img/demo.gif
new file mode 100644
index 0000000000..6144e6741d
Binary files /dev/null and b/blog/2025-01-07-inngest-ai-workflows/img/demo.gif differ
diff --git a/blog/2025-01-07-inngest-ai-workflows/img/hero.png b/blog/2025-01-07-inngest-ai-workflows/img/hero.png
new file mode 100644
index 0000000000..8f690462e5
Binary files /dev/null and b/blog/2025-01-07-inngest-ai-workflows/img/hero.png differ
diff --git a/blog/2025-01-07-inngest-ai-workflows/img/image1.png b/blog/2025-01-07-inngest-ai-workflows/img/image1.png
new file mode 100644
index 0000000000..7309a4165f
Binary files /dev/null and b/blog/2025-01-07-inngest-ai-workflows/img/image1.png differ
diff --git a/blog/2025-01-07-inngest-ai-workflows/img/image2.png b/blog/2025-01-07-inngest-ai-workflows/img/image2.png
new file mode 100644
index 0000000000..044c1622cc
Binary files /dev/null and b/blog/2025-01-07-inngest-ai-workflows/img/image2.png differ
diff --git a/blog/2025-01-07-inngest-ai-workflows/img/image3.png b/blog/2025-01-07-inngest-ai-workflows/img/image3.png
new file mode 100644
index 0000000000..70beaa807d
Binary files /dev/null and b/blog/2025-01-07-inngest-ai-workflows/img/image3.png differ
diff --git a/blog/2025-01-07-inngest-ai-workflows/index.mdx b/blog/2025-01-07-inngest-ai-workflows/index.mdx
new file mode 100644
index 0000000000..0a33b3e448
--- /dev/null
+++ b/blog/2025-01-07-inngest-ai-workflows/index.mdx
@@ -0,0 +1,279 @@
+---
+title: 'Building Agentic Workflows with Inngest'
+slug: inngest-ai-workflows
+description: "Automate your RAG pipelines and dive into Workflow Driven Agentic Applications that leverage Inngest and Weaviate."
+authors: [daniel, charly]
+date: 2025-01-07
+tags: []
+image: ./img/hero.png
+---
+
+
+
+---
+AI workflows are the backbone of all AI applications, whether you are building RAG pipelines, agentic workflows, or AI agents. Since 2023, most AI applications have relied on static AI workflows like RAG pipelines. In 2024, the concept of agentic workflows—*workflows with dynamic steps planned by LLMs*—surfaced to cope with the limitations of static workflows, such as hallucination, a lack of relevancy, or flexibility.
+
+It has been announced that 2025 will be the race for AI Agents, and having the right tools to build flexible and robust AI workflows will be crucial. For this reason, we are happy to partner with [Inngest](https://www.inngest.com/?utm_source=weaviate&utm_content=article&utm_campaign=workflow-dinner-generator), a workflow engine already used by tens of thousands of developers and handling over 100 million daily runs.
+
+This article will show you how to use Inngest and Weaviate to build an agentic RAG pipeline that generates the perfect dinner menu based on tastes, allergies, and the number of attendees.
+
+The finished demo is [available on GitHub](https://github.com/inngest/weaviate-agentic-workflow) so that you can follow the steps locally.
+
+## Step 1: Setup our Dinner Generator Agentic Workflow architecture
+
+Our Dinner Generator workflow relies on three datasets available on Huggingface:
+
+- `Thefoodprocessor/cuisine_type`: a dataset of pairs of recipes and cuisine types (ex: “Italian”)
+- `Thefoodprocessor/ingredients_alternatives`: a dataset of recipe and matching ingredients alternatives for known allergies
+- `Thefoodprocessor/wine_type`: a list of pairs of recipes and wine pairings
+
+Once those datasets are loaded into our Weaviate cluster, our workflow is triggered with an [Inngest Event](https://www.inngest.com/docs/features/events-triggers?utm_source=weaviate&utm_content=article&utm_campaign=workflow-dinner-generator) with the following properties:
+
+```
+{
+ "data": {
+ "participantsCount": 4,
+ "preferences": ["no nuts", "vegan"],
+ "cuisine_type": "Italian"
+ }
+}
+```
+
+Triggered by the above event, our Inngest agentic workflow goes through the following steps:
+
+
+
+
+1. First, our workflow will retrieve some recipes from the `Thefoodprocessorcuisinetype` Weaviate collection.
+2. Then, using the OpenAI `gpt-4` model, it will try to see if the food preferences contain some allergies.
+ 2.1 If some allergies are found, an extra step is dynamically performed to retrieve some ingredient alternatives from the `Thefoodprocessoringredientsalternatives` collection.
+3. Then, an OpenAI call is performed to generate a menu (with some ingredient alternatives, if any)
+4. Another weaviate collection query is performed against the `Thefoodprocessorwinetype` collection to find wines matching the generated menu.
+5. An additional OpenAI call is made to select the wine and update the dinner menu.
+6. Finally, a shopping list is generated from the final dinner menu.
+
+*The agentic nature of our dinner generator workflow resides in its ability to fetch additional information when allergies are mentioned in the food preferences.*
+
+The following section shows how Inngest and Weaviate make building such agentic and data-intensive workflows easy.
+
+## Step 2: Quick and easy embedding with Weaviate vectorizer
+
+[Huggingface datasets are available as `.parquet` files](https://huggingface.co/Thefoodprocessor), primarily designed for Python scripts.
+For this reason, [our repository](https://github.com/inngest/weaviate-agentic-workflow) contains the translated`.json` files to facilitate processing in Node.js.
+The `npm run load` script leverages Weaviate [batching](https://weaviate.io/developers/weaviate/client-libraries/typescript/typescript-v3#batch-inserts) and [vectorizer](https://weaviate.io/blog/typescript-client-stable-release#vectorizer-configuration) APIs to create new collections with a few lines of code:
+
+```javascript
+async function processJsonFile(
+ filePath: string,
+ client: any,
+ batchSize: number = 100
+) {
+ const jsonData = JSON.parse(await fs.readFile(filePath, "utf-8"));
+ let count = 0;
+ let records: any[] = [];
+
+ // Create collection name from filename (without extension)
+ const collectionName = path
+ .basename(filePath, ".json")
+ .replace(/[^a-zA-Z0-9]/g, "") // Remove special characters
+ .replace(/^[0-9]/, "C$&"); // Prefix with 'C' if starts with number
+
+ const properties = Object.keys(jsonData[0]);
+
+ // Create collection for this file
+ const collection = await client.collections.create({
+ name: collectionName,
+ properties: properties.map((property) => ({
+ name: property,
+ dataType: "text" as const,
+ })),
+ // Use Weaviate vectorize to create embeddings
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecOpenAI({
+ name: "recipe_vector",
+ sourceProperties: properties as any,
+ }),
+ ],
+ });
+
+ for (const record of jsonData) {
+ const { id: _, ...cleanRecord } = record;
+
+ records.push({
+ class: collectionName,
+ properties: cleanRecord,
+ });
+
+ count++;
+
+ // we push new records in batches using Weaviate's insertMany()
+ if (count % batchSize === 0) {
+ console.log(
+ `Processing batch from ${path.basename(filePath)}: ${
+ records.length
+ } records`
+ );
+ await collection.data.insertMany(records);
+ records = [];
+ }
+ }
+
+ // Process any remaining records
+ if (records.length > 0) {
+ console.log(
+ `Processing final batch from ${path.basename(filePath)}: ${
+ records.length
+ } records`
+ );
+ await collection.data.insertMany(records);
+ }
+
+ return { count, collectionName };
+}
+
+```
+
+When embedding thousands of records, Weaviate’s `vectorizer` and `insertMany()` APIs save us from handling rate limits with OpenAI’s embedding API.
+
+With our Weaviate collections ready, let’s look at the agentic workflow implementation.
+
+## Step 3: Combine Weaviate and Inngest DX to build our Agentic Workflow
+
+Inngest enables us to create AI workflows with a simple TypeScript API while Weaviate enable us to quickly retrieve data from our vectorized collections:
+
+```javascript
+import { openai } from "inngest";
+import { inngest } from "./inngest-client";
+import { weaviate } from "./weaviate-client";
+
+export const generateMeal = inngest.createFunction(
+ { id: "generate-meal", concurrency: 10 },
+ { event: "meal.generate" },
+ async ({ event, step }) => {
+ // retrieve the input parameters from our trigger event
+ const { participantsCount, preferences, cuisine_type } = event.data;
+
+ // Query relevant recipes based on cusine type preference
+ let relevantRecipes = await step.run("query-recipes", async () => {
+ const collection = weaviate.collections.get("Thefoodprocessorcuisinetype");
+ const query = cuisine_type;
+
+ const result = await collection.query.nearText(query, {
+ limit: 10,
+ });
+
+ return result.objects.map((r) => r.properties.recipe).join(`\n`);
+ });
+
+ const allergiesAnalysis = await step.ai.infer("Allergies analysis", {
+ model: openai({ model: "gpt-4" }),
+ body: {
+ messages: [
+ {
+ role: "system",
+ content:
+ 'Given the following list meal requirements, return a list of allergies, if not allergies, return "not allergies"',
+ },
+ {
+ role: "user",
+ content: preferences.join(", "),
+ },
+ ],
+ temperature: 0.7,
+ },
+ });
+
+ // ...
+});
+```
+
+*Please refer to the [repository’s README for instructions on setting up Inngest](https://github.com/inngest/weaviate-agentic-workflow).*
+An AI Workflow built with Inngest is composed of steps defined with `step.run()` or `step.ai.infer()`. With these two APIs, each step of our AI workflow will benefit from automatic retries, concurrency, and throttling control.
+Our Weaviate query, wrapped in `step.run()` will benefit from the `concurrency` configured in the Inngest Function definition, preventing any overload on our cluster.
+
+### Zooming on `step.ai.infer()`: a faster way to develop AI Workflows
+
+Using `step.ai.infer()` to perform LLM calls brings two significant improvements:
+
+1. The LLM requests get highlighted in the Inngest DevServer (*your local version of Inngest*), enabling you to **inspect the token usage, the input and output of the LLM**, and **re-run the given step with a new prompt**.
+
+ *Here is our `”Allergies analysis”` LLM step displayed in the Inngest Dev Server:*
+ 
+
+2. The LLM requests are **offloaded to Inngest’s infrastructure**, making it easier to deploy workflows on **serverless platforms such as Vercel**.
+
+### Agentic workflow steps as simple conditional statements
+
+Let’s come back to our workflow and add our dynamic query to the `Thefoodprocessoringredientsalternatives` Weaviate collection based on the result of our `”Allergies analysis”` LLM step.
+Achieving such agentic pattern with Inngest only takes simple conditional statements:
+
+```javascript
+import { openai } from "inngest";
+import { inngest } from "./inngest-client";
+import { weaviate } from "./weaviate-client";
+
+export const generateMeal = inngest.createFunction(
+ { id: "generate-meal", concurrency: 10 },
+ { event: "meal.generate" },
+ async ({ event, step }) => {
+ // retrieve the input parameters from our trigger event
+ const { participantsCount, preferences, cuisine_type } = event.data;
+
+ // Query relevant recipes based on cusine type preference
+ let relevantRecipes = await step.run("query-recipes", async () => {
+ // ...
+ });
+
+ // Look for allergies in the food preferences
+ const allergiesAnalysis = await step.ai.infer("Allergies analysis", {
+ // ...
+ });
+
+ let ingredientsAlternatives: any[] = [];
+
+ // run a conditional step based on `allergiesAnalysis` result
+ if (allergiesAnalysis.choices[0].message.content !== "not allergies") {
+ // Query relevant recipes based on cusine type preference
+ ingredientsAlternatives = await step.run(
+ "query-ingredients-alternatives",
+ async () => {
+ const collection = client.collections.get(
+ "Thefoodprocessoringredientsalternatives"
+ );
+ const query = allergiesAnalysis.choices[0].message.content!;
+
+ const result = await collection.query.nearText(query, {
+ limit: 10,
+ });
+
+ return result.objects;
+ }
+ );
+ }
+
+
+ // ...
+});
+```
+
+Starting our dinner generator workflow with food preferences including `”no nuts”` triggers the `"query-ingredients-alternatives"` Weaviate query step based on the detection of the `"Allergies analysis"` LLM step.
+
+
+
+The following steps of our Dinner generator workflow leverage Weaviate’s `collection.query.nearText()` and Inngest’s `step.ai.infer()` APIs similarly generate the menu, get the relevant wine pairing, and generate the shopping list.
+Let’s now see our Dinner generator in action.
+
+## Try it out: Generate a unique Dinner menu
+
+The above code and setup instructions are available in our recipes repository and can be run locally using the Inngest Dev Server.
+You’ll be able to generate multiple menu variations by trying different food preferences or cuisine types:
+
+
+
+This article covered how Weaviate’s rich embedding and query APIs make indexing and querying large vectorized datasets easy. It also demonstrated how Inngest’s composable API enables writing robust and dynamic workflows with simple APIs.
+
+This demo only scratches the surface of how combining Weaviate and Inngest can enable. If you want to build Agentic workflows with more autonomy, such as AI Agents, you should look at [Weaviate Generative Search](https://weaviate.io/developers/weaviate/model-providers/openai#generative-ai-models-for-rag) and Inngest’s new [AgentKit library](https://agentkit.inngest.com/overview). Or, you can wait for our next article on AI Agents.
+
+import WhatNext from '/_includes/what-next.mdx';
+
+
diff --git a/developers/academy/deployment/k8s/index.md b/developers/academy/deployment/k8s/index.md
index 0dba165467..4c023749ab 100644
--- a/developers/academy/deployment/k8s/index.md
+++ b/developers/academy/deployment/k8s/index.md
@@ -1,8 +1,12 @@
---
title: "Run Weaviate on Kubernetes"
+description: "Learn how to deploy Weaviate with Kubernetes, covering setup, management, and scalability tips."
sidebar_position: 100
---
+
+
+
import LearningGoals from '/src/components/Academy/learningGoals.jsx';
import CourseUnits from '/src/components/Academy/courseUnits.jsx';
import { courseData } from '/src/components/Academy/courseData.js'
diff --git a/developers/academy/js/_snippets/intro_next_steps_js.mdx b/developers/academy/js/_snippets/intro_next_steps_js.mdx
new file mode 100644
index 0000000000..9137857b82
--- /dev/null
+++ b/developers/academy/js/_snippets/intro_next_steps_js.mdx
@@ -0,0 +1,32 @@
+Congratulations! You have completed this introductory course on Weaviate.
+
+Now that you have completed this course, you may be interested in exploring our documentation or the Academy for more advanced courses.
+
+Some of our more popular resources include:
+
+### Documentation
+
+- How-to guides
+ - [How-to: manage data](/developers/weaviate/manage-data/index.md): Code examples for *create*, *update* and *delete* operations.
+ - [How-to: search](/developers/weaviate/search/index.md): Code examples for all types of search operations.
+ - [How-to: configure Weaviate](/developers/weaviate/configuration/index.md): Guides for configuring Weaviate, such as [PQ](/developers/weaviate/configuration/compression/pq-compression.md) and [BQ](/developers/weaviate/configuration/compression/bq-compression.md) compression, [backups](/developers/weaviate/configuration/backups.md) and [replication](/developers/weaviate/configuration/replication.md).
+- [Concepts guides](/developers/weaviate/concepts/index.md): Guides for understanding Weaviate's architecture and concepts.
+- [API reference](/developers/weaviate/api/index.md): Detailed information about Weaviate's APIs.
+
+### Academy
+
+- [Working with Text](/developers/academy/js/starter_text_data): Learn how to use work with text data in Weaviate.
+- [Which search is right for me?](/developers/academy/js/standalone/which_search): Learn about the different types of searches in Weaviate and when to use them.
+
+
+import CTASocials from './cta_socials.mdx';
+
+
+
+See you soon! 👋
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/_standalone/_want-stack/_snippets/index.ts b/developers/academy/js/_standalone/_want-stack/_snippets/index.ts
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/developers/academy/js/_standalone/_want-stack/index.md b/developers/academy/js/_standalone/_want-stack/index.md
new file mode 100644
index 0000000000..35bc081129
--- /dev/null
+++ b/developers/academy/js/_standalone/_want-stack/index.md
@@ -0,0 +1,62 @@
+---
+title: What is the WANT stack
+sidebar_position: 10
+---
+
+## What is the WANT
+
+### What is WANT
+
+Weaviate, AI, Next.js and Typescript.
+
+The want stack is collection of technologies that simplify the process of build AI Native Applications.
+
+
+#### Weaviate
+
+Weaviate is a vector database, it gives you these benefits
+- vector search
+- dynamic generative model selection (you don't get stuck)
+
+#### AI
+
+- this ties heavily into everything else, AI is at the center of AI native, leveraging the flexibility of weaviate and teh vastness of its integrations, you can bring AI to your application from embeddings, to generative models to rerankers or recommenders. Without any tie in. The goal is to give you the most flexibility without the burden on choosing
+
+#### Next.js (or Nuxt.js)
+
+This brings modern web development,
+
+
+#### Typescript
+
+Typescript.. add content
+
+
+
+### Building WANT applications
+
+- Resources
+- Starter templates
+- Learning
+
+
+[Embed Youtube Course]
+
+A vector search is also versatile. It can be used to search multiple data modalities (e.g. text, images, audio, etc.), and across multiple languages.
+
+
+## Learning objectives
+
+import LearningGoalsExp from '/src/components/Academy/learningGoalsExp.mdx';
+
+
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/_standalone/client-server/10_why-client-server.mdx b/developers/academy/js/_standalone/client-server/10_why-client-server.mdx
new file mode 100644
index 0000000000..526c59e34b
--- /dev/null
+++ b/developers/academy/js/_standalone/client-server/10_why-client-server.mdx
@@ -0,0 +1,58 @@
+---
+title: What is the client-server architecture
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+
+
+## What and Why
+
+### What is the client-server architecture
+
+A client-server application is one that runs on a client device while accessing information from a remote server.
+
+In the context of building applications with Weaviate, this means you would have a server as an intermediary to handle all interactions with your database as opposed to interacting with your database directly from your client application.
+
+This section aims to guide you through how to do that as you build applications with the Weaviate Typescript v3 client: [weaviate-client](https://www.npmjs.com/package/weaviate-client).
+
+
+
+
+
+### Why client-server
+
+
+Besides the requirements of running the [weaviate-client](https://weaviate.io/developers/weaviate/client-libraries/typescript/typescript-v3#node-support-only), the client-server architecture is reliably more secure than interactions directly from the client.
+
+Having a client-server approach means you can optimize your use of Weaviate by implementing one or more of the following..
+
+- Load Balancing
+- User and Session Management
+- Security related Middleware
+
+
+
+
+### Using the weaviate-client
+
+Install the client by following these instructions. A big benefit of using the new v3 client is the introduction of the gRPC protocol. A fast, more reliable platform to handle interactions with the database at scale. Unfortunately, gRPC does not support browser-based client development.
+
+The v3 client uses gRPC to connect to your Weaviate instance. The client supports Node.js, server-based development. It does not support browser-based web client development.
+
+
+In the next sections we'll look at how to build client-server applications with..
+- [Using backend web frameworks](./20_building-client-server.mdx)
+- [Using fullstack web frameworks](./30_fullstack.mdx)
+
+
+
+### Best Practices
+
+As mentioned, it is recommended to use these,
+
+As you build applications it is important to think about the following
+- Your API Tokens and where in your architecture they live
+- Sanitize and validate all user inputs
+- Implement CORS (Cross-Origin Resource Sharing) correctly
diff --git a/developers/academy/js/_standalone/client-server/20_building-client-server.mdx b/developers/academy/js/_standalone/client-server/20_building-client-server.mdx
new file mode 100644
index 0000000000..4c7c3863e9
--- /dev/null
+++ b/developers/academy/js/_standalone/client-server/20_building-client-server.mdx
@@ -0,0 +1,95 @@
+---
+title: Using backend web frameworks
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!./_snippets/20_backend.js';
+
+
+## Building backend web frameworks
+
+This approach involves having two separate tools. One to build you server application; ideally a backend framework and another to build your client application. For this example, we will be using [Express.js](https://expressjs.com/en/starter/hello-world.html)
+
+### Building a server
+
+#### 1. Initialize a Node.js application
+
+To build a server, we will use Express as an example.
+
+In a new directory, run
+```bash
+npm init
+```
+#### 2. Install relevant project dependencies including the weaviate-client
+
+```bash
+npm install express dotenv weaviate
+```
+
+#### 2. Create your project credentials including..
+- Your Weaviate cluster URL and API key
+- Your Cohere API key
+
+and add them to your `.env` file.
+
+
+
+
+
+
+ok..
+
+#### 2. Initialize Weaviate
+Paste the following in `config/weaviate.js`
+
+
+
+
+
+Ok..
+
+#### 2. Create your server
+
+
+
+
+
+ok..
+
+
+#### 5. Run your server with
+In your terminal type
+```bash
+node app.js
+```
+
+Your server should be running on `localhost:3000`
+
+
+
+### Adding a client application
+
+With our server built, we can now make a call from a client application. We'll can create a basic client application with HTML and Javascript.
+
+
+
+Alternatively, in [Hoppscotch](https://hoppscotch.io/),
+
+
diff --git a/developers/academy/js/_standalone/client-server/30_fullstack.mdx b/developers/academy/js/_standalone/client-server/30_fullstack.mdx
new file mode 100644
index 0000000000..63f813c132
--- /dev/null
+++ b/developers/academy/js/_standalone/client-server/30_fullstack.mdx
@@ -0,0 +1,203 @@
+---
+title: Using fullstack web frameworks
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+
+
+## Using Fullstack Frameworks
+
+This approach involves having a single tool to build both your server application and client application. In modern web development terms, such a tool is called a fullstack web frameworks. For this example, we will be using [Nuxt.js](nuxt.com/docs/getting-started/installation), and [Next.js](https://nextjs.org/)
+
+
+### Using Next.js
+
+#### 1. Create a Next.js application with the command below.
+
+```bash
+create-next-app –ts –app
+```
+#### 2. Install relevant project dependencies including the Weaviate client with the command below.
+
+```bash
+npm install weaviate-client
+```
+
+#### 2. Create your project credentials including..
+- Your Weaviate cluster URL and API key
+- Your Cohere API key
+
+... and add them to a `.env` file.
+
+#### 3. Initialize Weaviate
+
+Create a file in `utils/weaviate.ts` and paste the following code
+
+```ts
+import weaviate from "weaviate-client";
+
+const client = await weaviate.connectToWeaviateCloud(process.env.WEAVIATE_HOST_URL as string,{
+ authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_ADMIN_KEY as string),
+ headers: {
+ 'X-Cohere-Api-Key': process.env.COHERE_API_KEY as string
+ }
+ },
+);
+```
+
+#### 4. Create an action to run on the server
+
+Next in `./utils/action.ts`, paste the following code.
+```ts
+"use server";
+
+import { connectToDB } from './weaviate.ts'
+
+export async function vectorSearch(searchTerm: string) {
+
+ const myCollection = client.collections.get('MyCollectionName');
+
+ const response = await myCollection.query.nearText(searchTerm, {
+ limit: 8,
+ returnMetadata: ['distance'],
+ })
+
+ return response
+ }
+```
+
+#### 5. Fetch data from your server in your client application.
+In the `./app` folder, paste the following code in `page.tsx`
+```tsx
+
+import { vectorSearch } from '@/utils/action.ts';
+
+export default async function Home() {
+ const search = "water sports i can win a medal in"
+ const data = await vectorSearch(search);
+
+ return (
+
+
+
+ { data }
+
+
+
+ )
+
+}
+
+```
+
+et voila. Find a fully working application here to try out and build from as you see you fit.
+
+
+### Using Nuxt.js
+
+#### 1. Create a Nuxt.js application with the command below.
+```bash
+npx nuxi@latest init
+```
+#### 2. Install relevant project dependencies including the Weaviate client with the command below.
+```bash
+npm install weaviate-client dotenv zod tsc
+```
+#### 2. Create your project credentials including..
+- Your Weaviate cluster URL and API key
+- Your Cohere API key
+
+Add your env vars to your `..env` file and follow the [Nuxt.js](https://nuxt.com/docs/guide/directory-structure/env) convention on environment variables.
+
+#### 3. Create a API route to server requests from the client
+
+In `server/api` create a file called `search.ts`
+
+```ts
+
+import weaviate, { WeaviateClient } from "weaviate-client"
+import { z } from 'zod'
+
+export default defineLazyEventHandler(async () => {
+ const config = useRuntimeConfig()
+
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(config.weaviateHostURL,{
+ authCredentials: new weaviate.ApiKey(config.weaviateReadKey),
+ headers: {
+ 'X-Cohere-Api-Key': config.cohereApiKey,
+ }
+ }
+)
+
+const responseSchema = z.object({
+ query: z.string(),
+})
+
+async function vectorSearch(searchTerm:string) {
+ const myProductCollection = client.collections.get('TestProduct')
+
+const response = await myProductCollection.query.nearText(searchTerm, { limit : 10 })
+
+return response.objects
+
+
+}
+
+ return defineEventHandler<{query: { query: string } }>(async (event) => {
+
+ const result = await getValidatedQuery(event, body => responseSchema.safeParse(body))
+ if (!result.success)
+ throw result.error.issues
+
+ const searchTerm = result.data.query
+
+ return await vectorSearch(searchTerm)
+ })
+})
+
+```
+
+#### 4. Fetch data from your server in your client application.
+
+In `app.vue` paste the following code.
+
+```html
+
+
+ {{ searchResult }}
+
+
+
+
+
+```
+
+et voila.. Find a fully working application here to try out and build from as you see you fit.
+
+
+### Other frameworks
+
+
+Although only detailing these, you can build with Weaviate using the following fullstack frameworks.
+
+- Solid
+- Angular
+
+To mention a few.
+
+
diff --git a/developers/academy/js/_standalone/client-server/_snippets/20_backend.js b/developers/academy/js/_standalone/client-server/_snippets/20_backend.js
new file mode 100644
index 0000000000..7874679c27
--- /dev/null
+++ b/developers/academy/js/_standalone/client-server/_snippets/20_backend.js
@@ -0,0 +1,66 @@
+// START weaviate.js
+import weaviate from 'weaviate-client'
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+export const connectToDB = async () => {
+ try {
+ const client = await weaviate.connectToWeaviateCloud(process.env.WEAVIATE_HOST_URL,{
+ authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_READ_KEY),
+ headers: {
+ 'X-Cohere-Api-Key': process.env.COHERE_API_KEY || '',
+ }
+ }
+ )
+ console.log(`We are connected! ${await client.isReady()}`);
+ return client
+ } catch (error) {
+ console.error(`Error: ${error.message}`);
+ process.exit(1);
+ }
+};
+
+// END weaviate.js
+const dotEnv = ```
+// START .env
+COHERE_API_KEY=
+WEAVIATE_HOST_URL=
+WEAVIATE_ADMIN_KEY=
+WEAVIATE_READ_KEY=
+OPENAI_API_KEY=
+// END .env
+```
+
+// START app.js
+import express from 'express';
+import { connectToDB } from './config/weaviate.js';
+
+const app = express();
+const port = 3005
+
+const client = await connectToDB();
+
+app.get('/', async function(req, res, next) {
+ var searchTerm = req.query.searchTerm;
+
+ const wikipedia = client.collections.get("Wikipedia")
+
+ try {
+ const response = await wikipedia.query.nearText(searchTerm, {
+ limit: 5
+ })
+
+ res.send(response.objects)
+ } catch (error) {
+ console.error(`Error: ${error.message}`);
+ }
+ })
+
+app.listen(port, () => {
+ console.log(`App listening on port ${port}`)
+})
+
+
+
+// END app.js
\ No newline at end of file
diff --git a/developers/academy/js/_standalone/client-server/_snippets/30_fullstack.js b/developers/academy/js/_standalone/client-server/_snippets/30_fullstack.js
new file mode 100644
index 0000000000..769cd0def3
--- /dev/null
+++ b/developers/academy/js/_standalone/client-server/_snippets/30_fullstack.js
@@ -0,0 +1,71 @@
+// START weaviate.js
+
+import weaviate from 'weaviate-client'
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+export const connectToDB = async () => {
+ try {
+ const client = await weaviate.connectToWeaviateCloud(process.env.WEAVIATE_HOST_URL,{
+ authCredentials: new weaviate.ApiKey(process.env.WEAVIATE_READ_KEY),
+ headers: {
+ 'X-Cohere-Api-Key': process.env.COHERE_API_KEY || '',
+ }
+ }
+ )
+ console.log(`We are connected! ${await client.isReady()}`);
+ return client
+ } catch (error) {
+ console.error(`Error: ${error.message}`);
+ process.exit(1);
+ }
+};
+
+// END weaviate.js
+
+// .env
+`
+COHERE_API_KEY=
+WEAVIATE_HOST_URL=
+WEAVIATE_ADMIN_KEY=
+WEAVIATE_READ_KEY=
+OPENAI_API_KEY=
+`
+// END .env
+
+
+// START app.js
+import express from 'express';
+import { connectToDB } from './config/weaviate.js';
+
+const app = express();
+const port = 3005
+
+const client = await connectToDB();
+
+
+
+app.get('/', async function(req, res, next) {
+ var searchTerm = req.query.searchTerm;
+
+ const wikipedia = client.collections.get("Wikipedia")
+
+ try {
+ const response = await wikipedia.query.nearText(searchTerm, {
+ limit: 5
+ })
+
+ res.send(response.objects)
+ } catch (error) {
+ console.error(`Error: ${error.message}`);
+ }
+ })
+
+app.listen(port, () => {
+ console.log(`App listening on port ${port}`)
+})
+
+
+
+// END app.js
\ No newline at end of file
diff --git a/developers/academy/js/_standalone/client-server/index.md b/developers/academy/js/_standalone/client-server/index.md
new file mode 100644
index 0000000000..652457b00d
--- /dev/null
+++ b/developers/academy/js/_standalone/client-server/index.md
@@ -0,0 +1,25 @@
+---
+title: Building client-server Applications
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+
+
+## Overview
+
+It is recommended that you employ the client-server architecture as you build applications using the [weaviate-client](https://www.npmjs.com/package/weaviate-client).
+
+We will get into why this is and detail various paths you could follow as you build applications in Javascript
+
+
+
+We will go over..
+- What the client-server architecture is
+- Why we opt for the client-server approach
+- How to build client-server applications with..
+ - Fullstack frameworks and,
+ - Backend web frameworks.
+
+
diff --git a/developers/academy/js/_standalone/index.md b/developers/academy/js/_standalone/index.md
new file mode 100644
index 0000000000..13c1a747dc
--- /dev/null
+++ b/developers/academy/js/_standalone/index.md
@@ -0,0 +1,16 @@
+---
+title: Short units
+sidebar_position: 900
+---
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+import CourseUnits from '/src/components/Academy/courseUnits.jsx';
+import { courseData } from '/src/components/Academy/courseData.js'
+
+## Overview
+
+This section hosts short, standalone units that you can read independently of any other topics.
+
+## Units
+
+
diff --git a/developers/academy/js/_standalone/using-ml-models/10_embedding.mdx b/developers/academy/js/_standalone/using-ml-models/10_embedding.mdx
new file mode 100644
index 0000000000..824fabd41c
--- /dev/null
+++ b/developers/academy/js/_standalone/using-ml-models/10_embedding.mdx
@@ -0,0 +1,125 @@
+---
+title: Understanding Embedding models
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!./_snippets/10_embedding.ts';
+
+
+### What are embedding models
+
+Embedding models are machine learning models trained to represent information as an array of numbers, frequently referred to as vector embeddings. Vectors or vector embeddings are numeric representations of data that represent certain properties or features. This representation can be used to efficiently search through objects in a vector space.
+
+
+
+
+
+### When to use embedding models
+
+Embeddings are the worker horses behind modern search and Retrieval-Augmented Generation (RAG) applications. They are great for..
+
+- **Search:** Results of searches are ranked by the distance from an input query vector.
+- **Classification:** Items are classified by what category their vector representation is closest to.
+- **Recommendations:** Items with similar vector representations are recommended to users.
+
+
+### Applications of embedding models
+
+Embedding models, like most machine learning models are typically limited to one or more modalities.
+
+We use modality to describe the type of input or output that a machine learning model can process or interact with to run. Typically, embedding modals fall into two buckets, uni-modal or multimodal.
+
+
+
+
+- Uni-modal Embeddings
+
+Uni-modal embeddings represents a single modality in a multidimensional vector space. Examples of these are text to vector models by cohere or image to vector models
+
+
+
+
+- Multimodal Embeddings
+Multimodal embeddings represent multiple modalities in a multidimensional space. Allowing cross modal retrieval and clustering. CLIP is a popular multimodal model that can create embeddings of text, audio and video data.
+
+
+
+
+
+
+### Using embedding models in Weaviate
+
+Weaviate takes most of the complexity of generating and managing embeddings away! Weaviate is configured to support many different vectorizer models and vectorizer service providers. It also gives you the option of providing your own vectors.
+
+In Weaviate, vector embeddings power hybrid and semantic search.
+
+Lets walk through what its like to use embeddings in Weaviate.
+
+## Connect to a Weaviate instance
+
+
+
+
+
+Initialize your connection with Weaviate and add relevant env vars
+
+## Defining a collection with vectorizer code
+
+
+
+
+
+When creating a collection in Weaviate, we define what embedding model we want to use. In this example we use a uni-modal text to vector model by cohere to embed our data. We also have `sourceProperties`
+
+We have the option of picking what properties in our data we can create models from.
+
+We can also set multiple vectorizers, see [named vectors]
+
+
+## Importing data
+
+
+
+
+
+Once our collection is created, we import data. Our data is vectorized at import time and we can make semantic search queries on it.
+
+
+
+## making a search with vectors in responses
+
+
+
+
+
+
+Here we make a query and set `showVectors` as true so we can see the objects' vectors in our response. Read more about [search here](https://weaviate.io/developers/weaviate/concepts/search)
+
+
+
+Various embedding models differ when it comes to performance and ability, [you can read this article on embeddings](https://weaviate.io/blog/how-to-choose-an-embedding-model) so you have an idea of what to think about as you about picking one.
+
+
+
diff --git a/developers/academy/js/_standalone/using-ml-models/20_generative.mdx b/developers/academy/js/_standalone/using-ml-models/20_generative.mdx
new file mode 100644
index 0000000000..4ae925c319
--- /dev/null
+++ b/developers/academy/js/_standalone/using-ml-models/20_generative.mdx
@@ -0,0 +1,126 @@
+---
+title: Understanding Generative models
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!./_snippets/20_generative.ts';
+
+
+### What are generative models
+
+Generative models are machine learning models that when prompted, can generate original data guided by instructions in the prompt i.e. text, images, and other forms. This original data is derived from data it was trained on but does not mimic it like for like.
+
+
+
+Generative Models encompass so many types of models, we will specifically focus on large language models (LLMs).
+
+### When to use generative models
+
+Generative models are stars in the limelight of retrieval augmented generation (RAG) and agentic workflows. They are great for...
+
+- **Translation:** Models can perform zero-shot translate text from one language to another with extremely high accuracy.
+- **Code Generation:** Models can take high-level instructions and turn them into functional custom code.
+- **Image Generation:** Models can consistently generate high quality images from text instructions in a prompt.
+
+### Applications of generative models
+
+
+Large Language Models (LLMs), like Anthropics Claudes series of models, or Google's Gemini are specialized types of generative models focused on text data. These models, like most machine learning models are typically limited to one or more modalities.
+
+We use modality to describe the type of input or output that a machine learning model can process or interact with to run. Typically, generative modals fall into two buckets, uni-modal or multimodal.
+
+
+
+
+
+- Uni-modal generation
+
+In the context on LLMs, uni-modal generation defines a models ability to generate content and receive instructions in a single modality, this modality is usually text.
+
+
+
+
+- Multimodal Generation
+In the context on LLMs, multimodal generation defines a models ability to generate and receive instructions in multiple modalities. This can range from text input to generation or even image input to audio generation.
+
+
+
+
+### Using generative models in Weaviate
+
+Weaviate is configured to support many generative models and generative model providers. You can even plug in your own generative model too depending on where in the weaviate workflow you need generative capabilities.
+
+
+In Weaviate, generative models power our generative search. Lets walk through what its like to use generative models in Weaviate.
+
+## Connect to a Weaviate instance
+
+
+
+
+
+Initialize your connection with Weaviate and add relevant environment variables
+
+## Defining a collection with vectorizer code
+
+
+
+
+
+When creating a collection in Weaviate, we define what generative model we want to use. In this example we use a text generation model by Cohere to generate new data.
+
+## Importing data
+
+
+
+
+
+Once our collection is created, we import data. Our data is vectorized at import time and we can make semantic search queries on it. We can then use the results of these searches as context for our interactions with large language models.
+
+## Making a single prompt generative search
+
+
+
+
+
+Here we use a `singlePrompt` to make `n` requests to the langauge model where `n` is the number of responses we get from our search. We use `limit` to strictly define the number of responses we get. We can place responses from each response into our prompt with this format `{ answer }` i.e we want the answer property from our search response to be translated in french.
+
+## Making a grouped task generative search
+
+
+
+
+
+
+Here we use the `groupedTask` prompt format to group all the response from our search and send them alongside our prompt as context for what ever we are requesting. You can see with `groupedProperties` we only pass the answer property from all the results we get as context to the large language model, giving us control of what information will inform the models output.
+
+
+
+Various generative models differ when it comes to performance and ability, [you can take a look at our integrations page](https://weaviate.io/developers/weaviate/model-providers) to have a better idea of what options you can use.
\ No newline at end of file
diff --git a/developers/academy/js/_standalone/using-ml-models/_img/embedding_model-bae274b0162e29fa1df717a97a696bc3.png b/developers/academy/js/_standalone/using-ml-models/_img/embedding_model-bae274b0162e29fa1df717a97a696bc3.png
new file mode 100644
index 0000000000..4930ad4550
Binary files /dev/null and b/developers/academy/js/_standalone/using-ml-models/_img/embedding_model-bae274b0162e29fa1df717a97a696bc3.png differ
diff --git a/developers/academy/js/_standalone/using-ml-models/_snippets/10_embedding.ts b/developers/academy/js/_standalone/using-ml-models/_snippets/10_embedding.ts
new file mode 100644
index 0000000000..ee73bab376
--- /dev/null
+++ b/developers/academy/js/_standalone/using-ml-models/_snippets/10_embedding.ts
@@ -0,0 +1,83 @@
+// START Connect
+import weaviate, { WeaviateClient } from 'weaviate-client'
+import axios from 'axios';
+
+// END Connect
+import 'dotenv/config'
+
+// START Connect
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+const cohereKey = process.env.COHERE_API_KEY as string
+
+// Connect to your Weaviate instance
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL,{
+ authCredentials: new weaviate.ApiKey(weaviateKey),
+ headers: {
+ 'X-Cohere-Api-Key': cohereKey, // Replace with your inference API key
+ }
+ }
+)
+// END Connect
+ // Delete the "JeopardyQuestion" collection if it exists
+ await client.collections.delete('JeopardyQuestion');
+
+ if (await client.collections.exists('JeopardyQuestion') == false) {
+
+ // Create a collection with both a vectorizer and generative model
+// START Collection
+await client.collections.create({
+ name: 'JeopardyQuestion',
+ properties: [
+ {
+ name: 'Category',
+ dataType: 'text' as const,
+ description: 'Category of the question',
+ },
+ {
+ name: 'Question',
+ dataType: 'text' as const,
+ description: 'The question',
+ },
+ {
+ name: 'Answer',
+ dataType: 'text' as const,
+ description: 'The answer',
+ }
+ ],
+ // Define your Cohere vectorizer and generative model
+ vectorizers: weaviate.configure.vectorizer.text2VecCohere({
+ sourceProperties: ["Question", "Answer"]
+ }),
+});
+// END Collection
+
+ }
+
+ try {
+// START Importing
+let jeopardyCollection = client.collections.get('JeopardyQuestion');
+// Download data to import into the "JeopardyQuestion" collection
+const url = 'https://raw.githubusercontent.com/weaviate/weaviate-examples/main/jeopardy_small_dataset/jeopardy_tiny.json'
+const jeopardyQuestions = await axios.get(url);
+
+// Bulk insert downloaded data into the "JeopardyQuestion" collection
+await jeopardyCollection.data.insertMany(jeopardyQuestions.data)
+// END Importing
+ console.log('Data Imported');
+ } catch (e) {
+ console.error(e);
+ }
+// START Search
+const jeopardyCollection = client.collections.get('JeopardyQuestion');
+
+const searchResults = await jeopardyCollection.query.nearText(['question about animals'], {
+ limit: 3,
+ returnMetadata: ['distance'], // Return the distance of results from the query vector
+ includeVector: false // Change to true to include objects' vectors in your response
+})
+
+console.log("Near Text objects for:", JSON.stringify(searchResults, null, 2));
+// END Search
+
+
diff --git a/developers/academy/js/_standalone/using-ml-models/_snippets/20_generative.ts b/developers/academy/js/_standalone/using-ml-models/_snippets/20_generative.ts
new file mode 100644
index 0000000000..d0fd909f99
--- /dev/null
+++ b/developers/academy/js/_standalone/using-ml-models/_snippets/20_generative.ts
@@ -0,0 +1,93 @@
+// START Connect
+import weaviate, { WeaviateClient } from 'weaviate-client'
+import axios from 'axios';
+
+// END Connect
+
+import 'dotenv/config'
+
+// START Connect
+const weaviateURL = process.env.WEAVIATE_URL as string
+const weaviateKey = process.env.WEAVIATE_ADMIN_KEY as string
+const cohereKey = process.env.COHERE_API_KEY as string
+
+// Connect to your Weaviate instance
+const client: WeaviateClient = await weaviate.connectToWeaviateCloud(weaviateURL,{
+ authCredentials: new weaviate.ApiKey(weaviateKey),
+ headers: {
+ 'X-Cohere-Api-Key': cohereKey, // Replace with your inference API key
+ }
+ }
+)
+// END Connect
+
+ // Delete the "JeopardyQuestion" collection if it exists
+ await client.collections.delete('JeopardyQuestion');
+
+ if (await client.collections.exists('JeopardyQuestion') == false) {
+// START Collection
+await client.collections.create({
+ name: 'JeopardyQuestion',
+ properties: [
+ {
+ name: 'Category',
+ dataType: 'text' as const,
+ description: 'Category of the question',
+ },
+ {
+ name: 'Question',
+ dataType: 'text' as const,
+ description: 'The question',
+ },
+ {
+ name: 'Answer',
+ dataType: 'text' as const,
+ description: 'The answer',
+ }
+ ],
+ // Define your Cohere vectorizer and generative model
+ vectorizers: weaviate.configure.vectorizer.text2VecCohere(),
+ // highlight-start
+ generative: weaviate.configure.generative.cohere()
+ // highlight-start
+});
+// END Collection
+
+}
+
+ try {
+// START Importing
+let jeopardyCollection = client.collections.get('JeopardyQuestion');
+
+// Download data to import into the "JeopardyQuestion" collection
+const url = 'https://raw.githubusercontent.com/weaviate/weaviate-examples/main/jeopardy_small_dataset/jeopardy_tiny.json'
+const jeopardyQuestions = await axios.get(url);
+
+// Bulk insert downloaded data into the "JeopardyQuestion" collection
+await jeopardyCollection.data.insertMany(jeopardyQuestions.data)
+
+console.log('Data Imported');
+// END Importing
+ } catch (e) {
+ console.error(e);
+ }
+
+ const myCollection = client.collections.get('JeopardyQuestion');
+// START SingleGenerative
+const genResult = await myCollection.generate.nearText("african elephant in savanna", {
+ singlePrompt: "translate {answer} into french for me",
+})
+
+for (const item of genResult.objects) {
+ console.log("Single generated concept:", item.generated);
+}
+// END SingleGenerative
+
+// START GroupedGenerative
+const groupedGenResult = await myCollection.generate.nearText("african elephant in savanna", {
+ groupedTask: "Summarize all the results received into a single informational paragraph?",
+ groupedProperties: ["answer"]
+})
+
+console.log("Grouped generated concept:", groupedGenResult.generated);
+// END GroupedGenerative
diff --git a/developers/academy/js/_standalone/using-ml-models/index.md b/developers/academy/js/_standalone/using-ml-models/index.md
new file mode 100644
index 0000000000..d26e70ac18
--- /dev/null
+++ b/developers/academy/js/_standalone/using-ml-models/index.md
@@ -0,0 +1,41 @@
+---
+title: Picking ML Models
+sidebar_position: 10
+---
+
+## Unit overview
+
+
+Weaviate lets you interact with two types of machine learning models - generative models and embedding models.
+
+This serves to help you understand, on a basic level the functioning and utility of both types of models.
+We will explore scenarios where each model type thrives in and look at what things to consider when picking sub groups of models.
+
+Then, we will also discuss the role of each of these models in building AI-Native applications ranging from chatbots, and RAG Apps to Agentic systems.
+
+- **[Embedding Models](./10_embedding.mdx)**
+- **[Generative Models](./20_generative.mdx)**
+
+
+
+### Prerequisites
+
+- A Node environment with `weaviate-client` installed.
+- Familiarity with Weaviate's search capabilities.
+- Intermediate coding proficiency (e.g. Javascript)
+
+## Learning objectives
+
+import LearningGoalsExp from '/src/components/Academy/learningGoalsExp.mdx';
+
+
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/_standalone/which-search/05_review.mdx b/developers/academy/js/_standalone/which-search/05_review.mdx
new file mode 100644
index 0000000000..8912c9393c
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/05_review.mdx
@@ -0,0 +1,118 @@
+---
+title: Review of search types
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!./_snippets/05_review.ts';
+
+## Overview
+
+Weaviate offers three primary search types - namely **vector**, **keyword**, and **hybrid** searches. Let's briefly recap what they are, and how they work.
+
+### Code examples
+
+These code examples are runnable, with the [`v3` Weaviate Typescript client](/developers/weaviate/client-libraries/typescript/typescript-v3.mdx). Connect to the pre-configured demo instance of Weaviate with the following code, and try the examples below.
+
+
+
+
+
+
+
+
+
+## Vector search
+
+A vector search finds objects with the most similar vectors to the query vector.
+
+Because each vector is a numerical representation of the underlying object, a vector similarity can be thought of as a similarity in meaning. Therefore a vector search is also called "semantic search".
+
+In Weaviate, you can search for objects with similar vectors in any of the following ways:
+
+With a source medium (e.g. text or image):
+
+
+
+
+
+
+
+
+
+With a vector:
+
+
+
+
+
+
+
+
+
+With an existing Weaviate object:
+
+
+
+
+
+
+
+
+
+## Keyword search
+
+A keyword search finds objects whose keywords (i.e. tokens) are the most relevant to the keywords (i.e. tokens) of the query. The relevance is determined by the [BM25F algorithm](https://en.wikipedia.org/wiki/Okapi_BM25).
+
+Intuitively, the BM25F algorithm determines "relevance" by considering how often a keyword appears in each field of the object, relative to how commonly the keyword appears in the entire dataset.
+
+
+
+
+
+
+
+
+
+## Hybrid search
+
+A hybrid search combines the results of a vector search and a keyword search. This is done by performing both searches, and them combining the two search results with a "fusion" algorithm.
+
+
+
+
+
+
+
+
diff --git a/developers/academy/js/_standalone/which-search/10_strengths.mdx b/developers/academy/js/_standalone/which-search/10_strengths.mdx
new file mode 100644
index 0000000000..d7b1dd5b80
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/10_strengths.mdx
@@ -0,0 +1,203 @@
+---
+title: Strengths of each search type
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!./_snippets/10_strengths.ts';
+
+
+## Overview
+
+These different search types are offered because they each have different characteristics, and therefore different strengths.
+
+Let's explore the relative strengths of each search type.
+
+## Strengths of vector search
+
+### Robustness
+
+A vector is a numerical representation of the underlying object's meaning. As a result, a vector search is robust to any changes that don't affect the meaning of the object.
+
+More concretely, a vector of "cat", for example, will be similar to a vector of "kitten", "feline", and "pet", even though their spellings are very different.
+
+See this in action below, where we search for "cat" and "kitten" using vector search.
+
+
+
+
+
+
+
+
+
+You see that the results for "cat" and "kitten" are very similar.
+
+In other words, the vectors for "cat" and "kitten" are similar in meaning, because the model can "understand" meaning.
+
+
+
+Similarly, a vector of "cat" is similar to the vector of "cat" with a spelling mistake, such as "caat", or "catt".
+
+
+
+
+
+
+
+
+
+Here, the results are basically identical.
+
+
+
+This robustness is a key strength of vector search, as it means that the searcher does not need to know the exact words used in the dataset. This is particularly useful when the concepts being searched for are not well-defined, or when the searcher is not familiar with the dataset.
+
+### Versatility
+
+A vector search is also versatile. It can be used to search multiple data modalities (e.g. text, images, audio, etc.), and across multiple languages.
+
+This is because the vector is a numerical representation of the underlying object's meaning, and therefore the same vector can be used to represent the same meaning, regardless of the data modality or language.
+
+In fact, some models are capable of search across multiple data modalities, or multiple languages (or both!). This is made possible by using a model that can convert the data into comparable vectors, regardless of the data modality or language.
+
+
+
+## Strengths of keyword search
+
+### Exactitude
+
+Keyword search is ideal for situations where locating precise matches are required. This is particularly useful in scenarios where there exist exact terms to search for, such as academic research, searches through domain-specific data or technical troubleshooting.
+
+The ability to return results that precisely match the search terms ensures that users receive the most relevant information for their specific queries.
+
+More concretely, take a look at the example below, where we search for "imaging".
+
+
+
+
+
+
+
+
+
+And when we inspect the results:
+
+
+
+A search for "imaging" using a keyword search returns the one result that contains that specific word.
+
+
+## Strengths of hybrid search
+
+A key strength of hybrid search is its resiliency. Let's explore this in more detail.
+
+### Resiliency
+
+A hybrid search is resilient as it combines top results from both vector and keyword search. This helps to mitigate either search's shortcomings.
+
+Take a look at the hybrid search example below.
+
+
+
+
+
+
+
+
+
+We can inspect the results here:
+
+
+
+You can see that as well as the keyword search result above (for "imaging"), we get a semantically relevant result (for "X-rays").
+
+Because hybrid search combines the results of both vector and keyword search, it will find objects that score well on at least one of the search types. This approach has the effect of complementing each search type.
+
+
+
diff --git a/developers/academy/js/_standalone/which-search/20_selection.mdx b/developers/academy/js/_standalone/which-search/20_selection.mdx
new file mode 100644
index 0000000000..cb12999bc5
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/20_selection.mdx
@@ -0,0 +1,104 @@
+---
+title: Selecting the right search type
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!./_snippets/20_selection.ts';
+
+
+## Overview
+
+Selecting the right search type is key for effective search tasks. Let's explore how to select the right search type for your needs.
+
+
+## Rules of thumb
+
+
+### When to use vector search
+
+Vector search is the most robust and versatile search type. As such, it is well-suited for situations where the the meaning, or the vector representation, is of the highest importance.
+
+In cross-modal, object-based or multi-lingual searches, vector search may be the only viable option.
+
+Start with vector search for:
+
+- **Non-text, or cross-modal searches**: Essential for searching across different types of media, like finding images using text descriptions or vice versa.
+- **Object-based searches**: For finding similar objects to an extracted text chunk, image, or video, vector search is likely the only viable solution.
+- **Multi-lingual contexts**: The go-to choice for handling searches in multiple languages, where traditional keyword-based search may fall short.
+- **Complex query understanding**: Vector search excels in interpreting and responding to complex queries that require understanding context or nuances in language.
+
+### When to use keyword search
+
+Keyword search is useful when there is an expectation or requirement to match the exact search terms. This can be the case for specific domains such as legal, medical or technical areas where the exact terminology is important.
+
+Keyword search is also useful when the user is unlikely to make mistakes in inputs and is inputting a predictable set of terms, such as through a sanitized form or a drop-down menu.
+
+In summary, start with keyword search for:
+
+**Exact term matching**: Ideal in domains like legal, medical, or technical fields where specific terminology is crucial.
+**Predictable user inputs**: Works well when users are expected to input a defined set of terms, like through forms or drop-down menus.
+**Simple and direct queries**: Effective for straightforward search needs where the complexity of natural language processing is not required.
+**Fast and specific results**: Suitable for quick retrieval of information based on specific keywords or phrases.
+
+### When to use hybrid search
+
+Hybrid search is a great choice for "messy" situations.
+
+Because hybrid search combines results sets from both vector and keyword searches, it is able to provide a good balance between the robustness of vector search and the exactitude of keyword search.
+
+As a result, hybrid search is a generally good choice for most search needs that do not fall into the specific use cases of vector or keyword search.
+
+In summary, consider hybrid search for:
+
+- **Broad topic ranges**: Effective in scenarios where the target corpus covers a wide array of subjects, requiring a versatile search approach.
+- **Versatile search scenarios**: Useful for real-life scenarios that often require a combination of results from both vector and keyword searches.
+- **Unpredictable user inputs**: Ideal for many real-life scenarios where the user has free reign over the query. Some user queries may be aimed at direct matches while others' queries may be more about the overall meaning.
+
+
+
diff --git a/developers/academy/js/_standalone/which-search/30_strategies.mdx b/developers/academy/js/_standalone/which-search/30_strategies.mdx
new file mode 100644
index 0000000000..01e63e3001
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/30_strategies.mdx
@@ -0,0 +1,164 @@
+---
+title: Strategies to improve search results
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!./_snippets/30_strategies.ts';
+
+
+## Overview
+
+In addition to selecting the right search types, there are also strategies you can employ to improve the quality of your search results.
+
+Let's explore some of these strategies.
+
+## Improve vector search
+
+The key to improving vector search is to make sure that the vector representation of the object is fit for purpose, so as to suit the search needs.
+
+### Vectorizer selection
+
+Unless you are inserting data with your own vectors, you will be using a Weaviate vectorizer module, and a model within that module, to generate vectors for your data.
+
+The choice of vectorizer module and model is important, as it will determine what aspects of the data are captured in the vector representation, and how well the model is able to "understand" the data.
+
+First and foremost, you should select a vectorizer module that is best suited for your data type. For example, if you are working with text data, you should use the `text2vec` module, and if you are using image or multi-modal data, you should likely use the `multi2vec` module.
+
+We will cover vectorizer selection in another unit. But, if you are not sure where to start, try:
+- `text2vec-cohere`, or `text2vec-openai` for text data (API-based)
+ - Cohere offers a multi-lingual model that can be used with over 100 languages.
+- `multi2vec-clip` for image or image and text data.
+
+If you are working with text and prefer to run a local inference container, try `text2vec-transformers`, with a popular model such as `sentence-transformers/all-MiniLM-L12-v2`.
+
+### Try a re-ranker
+
+Re-ranker modules are a great way to improve the quality of your search results.
+
+A re-ranker module is a module that takes in the results of a vector search, and re-ranks the results based on additional criteria, or a different model. This allows a higher-quality (but slower) model to be used for re-ranking, while still benefiting from the fast first stage search.
+
+For example, you can use the `text2vec-cohere` module to perform a vector search, and then use the `reranker-cohere` module to re-rank the results using a different model.
+
+### Property selection
+
+Vectorization captures the "meaning" of the object. Accordingly, if a property is not relevant to the criteria to be applied for search, it should be excluded from the vectorization process.
+
+As an example, if a product object includes metadata such as its manufacturing process or location, and the vector search is intended to be based on the product's features, then the properties for manufacturing process and location should be excluded from the vectorization process.
+
+You can do this by specifying whether to skip a property during vectorization, as shown below. Note that you can do the same with the collection name, and the property name.
+
+
+
+
+
+
+
+
+
+### Chunking
+
+Chunking refers to the process of splitting a text into smaller chunks, and vectorizing each chunk separately. This is very important, as it defines how much information each vector contains.
+
+As a rule of thumb, the more granular the search needs, the smaller the chunk size should be. For example, if you are searching for specific concepts and ideas, you should chunk data into smaller units such as sentences or small windows of text. Alternatively, if you are searching for broader concepts, such as finding relevant chapters or books, you might chunk text accordingly.
+
+Read more about it in the [chunking unit](../../standalone/chunking/index.mdx) of Weaviate Academy.
+
+## Improve keyword search
+
+### Tokenization
+
+Although we refer to BM25 search as a "keyword" search, in reality the exact matches are for "tokens", rather than words. This is a different tokenization process to that used for generating vector embeddings, but instead, it is used to build the inverted index for BM25 searches and filtering.
+
+Accordingly, the tokenization process is very important, as it determines what tokens are used for matching.
+
+The available options are: `word`, `lowercase`, `whitespace`, and `field`. The default (`word`) might be sufficient for prose, but for text where exact matches including case and symbols are important, something like `whitespace` might be more appropriate.
+
+Available tokenization options:
+
+import TokenizationDefinition from '/_includes/tokenization_definition.mdx';
+
+
+
+You can set tokenization in the collection configuration.
+
+
+
+
+
+
+
+
+
+### Select and boost properties
+
+If you observe that matches in some properties are having too much of an impact, you can exclude them from the search, and/or boost the importance certain properties.
+
+For example, matches in the `description` property might be more important than matches in the `notes` property. You can specify this at query time.
+
+
+
+
+
+
+
+
+
+## Improve hybrid search
+
+### Alpha
+
+The alpha parameter determines the balance between the vector and keyword search results.
+
+If you want to configure your search to be more vector-based, you can increase the alpha value. Conversely, if you want to configure your search to be more keyword-based, you can decrease the alpha value.
+
+
+
+
+
+
+
+
+
+### Fusion algorithm
+
+The fusion algorithm determines how the results from the vector and keyword searches are combined.
+
+By default, an inverse of the ranks from each results set are summed, in what is called the "ranked fusion" algorithm. However, you can also use the "relative score fusion" algorithm, which sums normalized scores from each results set.
+
+Generally, we have found that the "relative score fusion" algorithm works better, but you should try both to see which works best for your use case.
+
+
+
+
+
+
+
+
+
diff --git a/developers/academy/js/_standalone/which-search/_30_improve_search.mdx b/developers/academy/js/_standalone/which-search/_30_improve_search.mdx
new file mode 100644
index 0000000000..2447a084e3
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/_30_improve_search.mdx
@@ -0,0 +1,13 @@
+---
+title: Improving search
+---
+
+- Evaluating search quality
+- Improving search quality
+ - Vectorizer
+ - Data structure
+ - Search type
+- Improving search speed
+ - Inference speed
+ - Resource demands
+ - Query bottlenecks
diff --git a/developers/academy/js/_standalone/which-search/_img/multimodal_example.png b/developers/academy/js/_standalone/which-search/_img/multimodal_example.png
new file mode 100644
index 0000000000..628d4d5be1
Binary files /dev/null and b/developers/academy/js/_standalone/which-search/_img/multimodal_example.png differ
diff --git a/developers/academy/js/_standalone/which-search/_snippets/05_review.ts b/developers/academy/js/_standalone/which-search/_snippets/05_review.ts
new file mode 100644
index 0000000000..8df9d034f4
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/_snippets/05_review.ts
@@ -0,0 +1,112 @@
+import weaviate, { WeaviateClient, WeaviateReturn } from "weaviate-client"
+// START connectionCode
+let client: WeaviateClient
+
+
+client = await weaviate.connectToWeaviateCloud(process.env.WCD_URL as string, {
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+ headers: {
+ 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, // Replace with your inference API key
+ }
+}
+)
+// END connectionCode
+
+
+// START nearTextExample // START nearVectorExample // START nearObjectExample // START bm25Example // START hybridExample
+type NonGenericReturn = WeaviateReturn
+let response: NonGenericReturn
+
+const questions = client.collections.get("JeopardyQuestion")
+
+// END nearTextExample // END nearVectorExample // END nearObjectExample // END bm25Example // END hybridExample
+
+// START nearTextExample
+response = questions.query.nearText("space travel", // Your query string
+ {
+ limit: 2
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.uuid)
+ console.log(item.properties)
+}
+// END nearTextExample
+// assert len(response.objects) == 2
+// assert "question" in response.objects[0].properties.keys()
+
+response = questions.query.nearText("space travel", {
+ limit: 1,
+ includeVector: true
+}
+)
+let vectorInput = response.objects[0].vectors
+let objectInput = response.objects[0].uuid
+
+// START nearVectorExample
+response = questions.query.nearVector(vectorInput, // Your vectors
+ {
+ limit: 2
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.uuid)
+ console.log(item.properties)
+}
+// END nearVectorExample
+
+// assert len(response.objects) == 2
+// assert "question" in response.objects[0].properties.keys()
+
+// START nearObjectExample
+response = questions.query.nearObject(objectInput, // Your object UUID
+ {
+ limit: 2
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.uuid)
+ console.log(item.properties)
+}
+// END nearObjectExample
+
+
+// assert len(response.objects) == 2
+// assert "question" in response.objects[0].properties.keys()
+
+// START bm25Example
+response = questions.query.bm25("space travel", // Your query string
+ {
+ limit: 2
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.uuid)
+ console.log(item.properties)
+}
+// END bm25Example
+
+// assert len(response.objects) == 2
+// assert "question" in response.objects[0].properties.keys()
+
+
+// START hybridExample
+response = questions.query.hybrid("space travel", // Your query string
+ {
+ limit: 2
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.uuid)
+ console.log(item.properties)
+}
+
+// END hybridExample
+
+// assert len(response.objects) == 2
+// assert "question" in response.objects[0].properties.keys()
diff --git a/developers/academy/js/_standalone/which-search/_snippets/10_strengths.ts b/developers/academy/js/_standalone/which-search/_snippets/10_strengths.ts
new file mode 100644
index 0000000000..994c0ca581
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/_snippets/10_strengths.ts
@@ -0,0 +1,164 @@
+import weaviate, { WeaviateClient, WeaviateReturn } from "weaviate-client"
+
+let client: WeaviateClient
+type NonGenericReturn = WeaviateReturn
+let response: NonGenericReturn
+// # END-ANY
+
+client = await weaviate.connectToWeaviateCloud(process.env.WCD_URL as string,{
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+ headers: {
+ 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, // Replace with your inference API key
+ }
+ }
+ )
+
+// START robustnessExampleWords
+for (const query of ["cat", "kitten"]) {
+ const question = client.collections.get("JeopardyQuestion")
+
+ response = await question.query.nearText(query,{
+ limit: 1,
+ returnMetadata: ['distance'],
+ returnProperties: ["question", "answer"]
+ })
+
+ for (const item of response.objects) {
+ console.log(`\n===== Search results for ${query} =====`)
+ console.log("Distance:", item.metadata?.distance)
+ console.log(item.properties)
+ }
+}
+// END robustnessExampleWords
+// assert length(response.objects) == 1
+// assert "question" in response.objects[0].properties.keys()
+
+
+const example = `
+// START responseRobustnessExampleWords
+===== Search results for cat =====
+Distance: 0.170
+{
+ "answer": "Fat cat",
+ "question": "A flabby tabby"
+}
+
+===== Search results for kitten =====
+Distance: 0.150
+{
+ "answer": "Fat cat",
+ "question": "A flabby tabby"
+}
+// END responseRobustnessExampleWords
+`
+
+// START robustnessExampleSpelling
+for (const query of ["cat", "catt", "caat"]) {
+ const question = client.collections.get("JeopardyQuestion")
+
+ response = await question.query.nearText(query,{
+ limit: 1,
+ returnMetadata: ['distance'],
+ returnProperties: ["question", "answer"]
+ })
+
+ for (const item of response.objects) {
+ console.log(`\n===== Search results for ${query} =====`)
+ console.log("Distance:", item.metadata?.distance)
+ console.log(item.properties)
+ }
+}
+
+// END robustnessExampleSpelling
+
+// assert len(response.objects) == 1
+// assert "question" in response.objects[0].properties.keys()
+
+
+const example2 = `
+// START responseRobustnessExampleSpelling
+===== Search results for cat =====
+Distance: 0.170
+{
+ "answer": "Fat cat",
+ "question": "A flabby tabby"
+}
+
+===== Search results for catt =====
+Distance: 0.177
+{
+ "answer": "Fat cat",
+ "question": "A flabby tabby"
+}
+
+===== Search results for caat =====
+Distance: 0.184
+{
+ "answer": "Fat cat",
+ "question": "A flabby tabby"
+}
+// END responseRobustnessExampleSpelling
+`
+
+// START bm25Example // START hybridExample
+const question = client.collections.get("JeopardyQuestion")
+// END bm25Example // END hybridExample
+
+// START bm25Example
+
+response = await question.query.bm25("imaging",{ // Your query string
+ limit: 2,
+ returnProperties: ["question", "answer"]
+ })
+
+for (const item of response.objects) {
+ console.log(item.uuid)
+ console.log(item.properties)
+}
+// END bm25Example
+
+// assert "question" in response.objects[0].properties.keys()
+
+const example3 = `
+// START bm25Results
+49fe3d7c-61a5-5916-99bb-052d07c7c251
+{
+ "answer": "magnetic resonance imaging",
+ "question": "MRI, which stands for this, cannot be used on patients with pacemakers or artificial metal joints"
+}
+// END bm25Results
+`
+
+// START hybridExample
+
+response = await question.query.hybrid("imaging", { // Your query string
+ limit: 2,
+ returnMetadata: ["score"],
+ returnProperties: ["question", "answer"]
+ })
+
+for (const item of response.objects) {
+ console.log(item.uuid)
+ console.log(item.metadata?.score)
+ console.log(item.properties)
+}
+
+// END hybridExample
+
+// assert "question" in response.objects[0].properties.keys()
+
+
+`
+// START hybridResults
+49fe3d7c-61a5-5916-99bb-052d07c7c251
+{
+ "answer": "magnetic resonance imaging",
+ "question": "MRI, which stands for this, cannot be used on patients with pacemakers or artificial metal joints"
+}
+9041bce6-b5d1-5637-bcbe-2ebb8a689fe0
+{
+ "answer": "X-rays",
+ "question": "These electromagnetic rays used to take pictures of your insides were originally known as Roentgen rays"
+}
+// END hybridResults
+`
diff --git a/developers/academy/js/_standalone/which-search/_snippets/20_selection.ts b/developers/academy/js/_standalone/which-search/_snippets/20_selection.ts
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/developers/academy/js/_standalone/which-search/_snippets/30_strategies.ts b/developers/academy/js/_standalone/which-search/_snippets/30_strategies.ts
new file mode 100644
index 0000000000..94d1523744
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/_snippets/30_strategies.ts
@@ -0,0 +1,127 @@
+import weaviate, { CollectionConfig, CollectionConfigCreate, WeaviateClient, WeaviateReturn } from "weaviate-client"
+
+let client: WeaviateClient
+
+client = await weaviate.connectToWeaviateCloud(process.env.WCD_URL as string,{
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+ headers: {
+ 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY as string, // Replace with your inference API key
+ }
+ }
+)
+
+client.collections.delete("Product")
+
+// START skipVectorizationExample // START tokenizationExample
+let products: CollectionConfigCreate
+// END skipVectorizationExample // END tokenizationExample
+
+// START skipVectorizationExample
+products = await client.collections.create({
+ name: "Product",
+ vectorizers: weaviate.configure.vectorizer.text2VecOpenAI({
+ // highlight-start
+ vectorizeCollectionName: true
+ // highlight-end
+ }),
+ properties: [{
+ name: "name",
+ dataType: weaviate.configure.dataType.TEXT,
+ // highlight-start
+ vectorizePropertyName: true
+ // highlight-end
+ },
+ {
+ name: "description",
+ dataType: weaviate.configure.dataType.TEXT,
+ },
+ {
+ name: "manufacturing_process",
+ dataType: weaviate.configure.dataType.TEXT,
+ // highlight-start
+ skipVectorization: true, // Skip unwanted property
+ //highlight-end
+ }]
+})
+// END skipVectorizationExample
+
+client.collections.delete("Product")
+
+client.collections.delete("SomeCollection")
+
+// START tokenizationExample
+
+products = await client.collections.create({
+ name: "SomeCollection",
+ properties: [{
+ name: "name",
+ dataType: weaviate.configure.dataType.TEXT,
+ // highlight-start
+ tokenization: weaviate.configure.tokenization.WORD // Default
+ // highlight-end
+ },
+ {
+ name: "description",
+ dataType: weaviate.configure.dataType.TEXT,
+ tokenization: weaviate.configure.tokenization.WHITESPACE // Will keep case & special characters
+ },
+ {
+ name: "email",
+ dataType: weaviate.configure.dataType.TEXT,
+ // highlight-start
+ tokenization: weaviate.configure.tokenization.FIELD // Do not tokenize at all
+ // highlight-end
+ }]
+})
+// END tokenizationExample
+
+
+client.collections.delete("SomeCollection")
+
+// START selectAndBoostExample // START adjustAlpha // START changeFusionType
+type NonGenericReturn = WeaviateReturn
+let response: NonGenericReturn
+
+const questions = client.collections.get("JeopardyQuestion")
+
+// END selectAndBoostExample // END adjustAlpha // END changeFusionType
+
+
+// START selectAndBoostExample
+response = questions.query.bm25("animal",{ // Your query string
+ limit: 5,
+ queryProperties: ["question^3", "answer"] // Boost the impact of "question" property by 3
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.properties)
+}
+// END selectAndBoostExample
+
+// START adjustAlpha
+response = questions.query.hybrid("imaging",{ // Your query string
+ limit: 5,
+ alpha: 0.1, // Mostly a vector search (Try it with alpha=0.9)
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.properties)
+}
+// END adjustAlpha
+
+
+
+// START changeFusionType
+response = questions.query.hybrid("imaging",{ // Your query string
+ limit: 5,
+ fusionType: "RelativeScore",
+ alpha: 0.1, // Mostly a vector search (Try it with alpha=0.9)
+ }
+)
+
+for (const item of response.objects) {
+ console.log(item.properties)
+}
+// END changeFusionType
diff --git a/developers/academy/js/_standalone/which-search/index.md b/developers/academy/js/_standalone/which-search/index.md
new file mode 100644
index 0000000000..6b81f63eec
--- /dev/null
+++ b/developers/academy/js/_standalone/which-search/index.md
@@ -0,0 +1,50 @@
+---
+title: Which search is right for me?
+sidebar_position: 10
+---
+
+## Unit overview
+
+
+
+
+
+
+
+Weaviate offers three distinct search methods - namely **vector**, **keyword**, and **hybrid** searches.
+
+Each method has its unique strengths and applicabilities, making the selection critical to the success of your search-related tasks.
+
+This section compares these search types to equip you with the knowledge to intuit when and why to employ each of these search methodologies.
+
+We will explore how the choice of search type impacts not only the quality of the search results but also the overall performance of the search operation.
+
+Then, we will also discuss strategies to improve the quality of search results, as well as the performance of the search operation.
+
+
+### Prerequisites
+
+- A Node.js environment with `weaviate-client` installed.
+- Familiarity with Weaviate's search capabilities.
+- Intermediate coding proficiency (e.g. Javascript).
+
+## Learning objectives
+
+import LearningGoalsExp from '/src/components/Academy/learningGoalsExp.mdx';
+
+
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/101_setup_weaviate/10_client.mdx b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/10_client.mdx
new file mode 100644
index 0000000000..67bc24408c
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/10_client.mdx
@@ -0,0 +1,37 @@
+---
+title: Weaviate Typescript client
+description: Client Setup for Multimodal Data in Weaviate
+---
+
+## Installation
+
+The latest Weaviate TypeScript client library can be installed using npm. The client library is tested on Node v18 and later. Install it using the following command:
+
+```bash
+npm install weaviate-client
+```
+
+The latest major version is `v3` (e.g. `3.x.x`). You can check the version like so:
+
+```bash
+npm view weaviate-client version
+```
+
+## Basic usage
+
+You can import the Weaviate client library like so:
+
+```typescript
+import weaviate, { generateUuid5, ApiKey } from "weaviate-client"
+```
+
+The client provides sets of helper functions (e.g. `generateUuid5, ApiKey`) to make it easier to interact with Weaviate.
+
+Next, we'll show you how create a Weaviate instance and connect to it.
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/10_create_wcs.mdx b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/10_create_wcs.mdx
new file mode 100644
index 0000000000..edb85e0d90
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/10_create_wcs.mdx
@@ -0,0 +1,67 @@
+---
+title: "Option 1: A cloud WCD instance"
+description: "Create a Weaviate instance on WCS for scalable, cloud-based data projects."
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../../_snippets/101_connect.mts';
+
+Here, you will create a Weaviate Cloud (WCD) instance. WCD is a fully managed Weaviate instance that runs in the cloud. It's a great way to get started with Weaviate, as it requires no installation or maintenance.
+
+### Log in to the WCD Console
+
+Go to the [WCD Console](https://console.weaviate.cloud/) and log in with your credentials. If you don't have an account yet, you can sign up by clicking on the Register here link from the login screen.
+
+### Create a Weaviate instance
+
+From the console, go to the Dashboard and click on the Create cluster button. From the following screen:
+
+- Select the "Free sandbox" tab
+- Provide a cluster name
+- Set "Enable authentication" to "Yes"
+
+Click on the Create button to create your Weaviate instance. The process will take a few minutes.
+
+### Retrieve your Weaviate instance details
+
+Once the instance is created, you will be able see its details by clicking on the Details button. Find the cluster URL and the API key.
+
+You will need these details to connect to your Weaviate instance.
+
+### Connect to your WCD instance
+
+To connect to the Weaviate Cloud (WCD) instance, you need to use the cluster URL and the API key. You can find these details in the WCD Console.
+
+Use the `connectToWeaviateCloud()` function to connect to your WCD instance.
+
+
+
+#### Provide inference API keys
+
+Some Weaviate modules can use inference APIs for vectorizing data or large language model integration. You can provide the API keys for these services to Weaviate at instantiation.
+
+This course uses VoyageAI, so you can provide the VoyageAI API key to Weaviate through `headers: {"X-VoyageAI-Api-Key": }` as shown below:
+
+
+
+:::note What next?
+If you have completed this, you can skip the next page [Option 2: A local Weaviate instance](./20_create_docker.mdx) and continue with [Communicate with Weaviate](../30_communicate.mdx).
+:::
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/20_create_docker.mdx b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/20_create_docker.mdx
new file mode 100644
index 0000000000..af3632a709
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/20_create_docker.mdx
@@ -0,0 +1,93 @@
+---
+title: "Option 2: A local Docker instance"
+description: "Set up Weaviate with Docker for a quick and customizable local deployment."
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../../_snippets/101_connect.mts';
+
+:::note Have you already created a Weaviate instance?
+If you have [created a cloud instance](./10_create_wcs.mdx) of Weaviate, you can skip this page and continue with [Communicate with Weaviate](../30_communicate.mdx).
+:::
+
+Here, you will create a Weaviate instance using Docker.
+
+### Download and run the docker-compose file
+
+Install Docker on your machine. We recommend following the [official Docker installation guide](https://docs.docker.com/get-docker/).
+
+Create a new directory and navigate to it in your terminal. Then, create a new file called `docker-compose.yml` and add the following content:
+
+```yaml
+---
+version: '3.4'
+services:
+ weaviate_anon:
+ command:
+ - --host
+ - 0.0.0.0
+ - --port
+ - '8080'
+ - --scheme
+ - http
+ image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version||
+ ports:
+ - 8080:8080
+ - 50051:50051
+ restart: on-failure:0
+ environment:
+ COHERE_APIKEY: $COHERE_APIKEY
+ VOYAGEAI_APIKEY: $VOYAGEAI_APIKEY
+ QUERY_DEFAULTS_LIMIT: 25
+ AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+ PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+ DEFAULT_VECTORIZER_MODULE: 'none'
+ ENABLE_MODULES: 'text2vec-cohere,multi2vec-voyageai,generative-cohere'
+ BACKUP_FILESYSTEM_PATH: '/var/lib/weaviate/backups'
+ CLUSTER_HOSTNAME: 'node1'
+...
+```
+
+### Create a Weaviate instance
+
+Run the following command to start Weaviate:
+
+```bash
+docker compose up -d
+```
+
+### Your Weaviate instance details
+
+Once the instance is created, you can access it at `http://localhost:8080`.
+
+### Connect to your Weaviate instance
+
+To connect to the Weaviate instance, use the `connectToLocal()` function.
+
+
+
+#### Provide inference API keys
+
+Some Weaviate modules can use inference APIs for vectorizing data or large language model integration. You can provide the API keys for these services to Weaviate at instantiation.
+
+This course uses VoyageAI, so you can provide the VoyageAI API key to Weaviate through `headers: {"X-VoyageAI-Api-Key": }` as shown below:
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/index.mdx b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/index.mdx
new file mode 100644
index 0000000000..53c348780e
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/20_create_instance/index.mdx
@@ -0,0 +1,19 @@
+---
+title: Create a Weaviate instance
+description: "Create a new Weaviate instance tailored for text data applications."
+---
+
+For this unit, you can choose to create a Weaviate Cloud (WCD) instance or a local Docker instance.
+
+- [Create a Weaviate Cloud (WCD) instance](./10_create_wcs.mdx)
+ - If you want a managed service and don't want to worry about installation and maintenance.
+- [Create a local Docker instance](./20_create_docker.mdx)
+ - If you want to run Weaviate on your local machine, or want to have full control over the installation and maintenance.
+
+Either option is fine for this course. If you're not sure which to choose, we recommend starting with a WCD instance.
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/101_setup_weaviate/30_communicate.mdx b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/30_communicate.mdx
new file mode 100644
index 0000000000..9f190e9a82
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/30_communicate.mdx
@@ -0,0 +1,64 @@
+---
+title: Communicate with Weaviate
+description: Communication Setup for Multimodal Data
+---
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/101_connect.mts';
+
+Here, we'll perform basic operations to communicate with Weaviate using the TypeScript client library.
+
+### Check Weaviate status
+
+You can check whether the Weaviate instance is up using the `isLive` function.
+
+
+
+### Retrieve server meta information
+
+You can retrieve meta information about the Weaviate instance using the `getMeta` function.
+
+
+
+This will print the server meta information to the console. The output will look similar to the following:
+
+
+ Example getMeta() output
+
+
+
+
+### Close the connection
+
+After you have finished using the Weaviate client, you should close the connection. This frees up resources and ensures that the connection is properly closed.
+
+We suggest using a `try`-`finally` block as a best practice. For brevity, we will not include the `try`-`finally` blocks in the remaining code snippets.
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
\ No newline at end of file
diff --git a/developers/academy/js/starter_multimodal_data/101_setup_weaviate/index.mdx b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/index.mdx
new file mode 100644
index 0000000000..0c72f66c40
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/101_setup_weaviate/index.mdx
@@ -0,0 +1,26 @@
+---
+title: Set up Weaviate
+description: Weaviate Setup for Multimodal Data
+---
+
+
+
+
+
+
+
+
+## Learning objectives
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/102_mm_collections/10_preparation.mdx b/developers/academy/js/starter_multimodal_data/102_mm_collections/10_preparation.mdx
new file mode 100644
index 0000000000..55ff47bc9d
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/102_mm_collections/10_preparation.mdx
@@ -0,0 +1,36 @@
+---
+title: Preparation
+---
+
+In this section you are going to populate your Weaviate instance with a movie dataset, using the multi-modal, Cohere Embed 3 models to embed the text and image data.
+
+### Weaviate instance
+
+Make sure to have your Weaviate instance set up. You should have [created an instance](../101_setup_weaviate/20_create_instance/index.mdx) and be able to connect to it.
+
+### Source data
+
+We are going to use a movie dataset sourced from [TMDB](https://www.themoviedb.org/). The dataset can be found in this [GitHub repository](https://raw.githubusercontent.com/weaviate-tutorials/edu-datasets/main/movies_data_1990_2024.json), and it contains bibliographic information on ~700 movies released between 1990 and 2024.
+
+As a multimodal project, we'll also use [corresponding posters for each movie](https://raw.githubusercontent.com/weaviate-tutorials/edu-datasets/main/movies_data_1990_2024_posters.zip), which are available in the same repository.
+
+
+ See sample text data
+
+| | backdrop_path | genre_ids | id | original_language | original_title | overview | popularity | poster_path | release_date | title | video | vote_average | vote_count |
+|---:|:---------------------------------|:----------------|-----:|:--------------------|:----------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------:|:---------------------------------|:---------------|:----------------------------|:--------|---------------:|-------------:|
+| 0 | /3Nn5BOM1EVw1IYrv6MsbOS6N1Ol.jpg | [14, 18, 10749] | 162 | en | Edward Scissorhands | A small suburban town receives a visit from a castaway unfinished science experiment named Edward. | 45.694 | /1RFIbuW9Z3eN9Oxw2KaQG5DfLmD.jpg | 1990-12-07 | Edward Scissorhands | False | 7.7 | 12305 |
+| 1 | /sw7mordbZxgITU877yTpZCud90M.jpg | [18, 80] | 769 | en | GoodFellas | The true story of Henry Hill, a half-Irish, half-Sicilian Brooklyn kid who is adopted by neighbourhood gangsters at an early age and climbs the ranks of a Mafia family under the guidance of Jimmy Conway. | 57.228 | /aKuFiU82s5ISJpGZp7YkIr3kCUd.jpg | 1990-09-12 | GoodFellas | False | 8.5 | 12106 |
+| 2 | /6uLhSLXzB1ooJ3522ydrBZ2Hh0W.jpg | [35, 10751] | 771 | en | Home Alone | Eight-year-old Kevin McCallister makes the most of the situation after his family unwittingly leaves him behind when they go on Christmas vacation. But when a pair of bungling burglars set their sights on Kevin's house, the plucky kid stands ready to defend his territory. By planting booby traps galore, adorably mischievous Kevin stands his ground as his frantic mother attempts to race home before Christmas Day. | 3.538 | /onTSipZ8R3bliBdKfPtsDuHTdlL.jpg | 1990-11-16 | Home Alone | False | 7.4 | 10599 |
+| 3 | /vKp3NvqBkcjHkCHSGi6EbcP7g4J.jpg | [12, 35, 878] | 196 | en | Back to the Future Part III | The final installment of the Back to the Future trilogy finds Marty digging the trusty DeLorean out of a mineshaft and looking for Doc in the Wild West of 1885. But when their time machine breaks down, the travelers are stranded in a land of spurs. More problems arise when Doc falls for pretty schoolteacher Clara Clayton, and Marty tangles with Buford Tannen. | 28.896 | /crzoVQnMzIrRfHtQw0tLBirNfVg.jpg | 1990-05-25 | Back to the Future Part III | False | 7.5 | 9918 |
+| 4 | /3tuWpnCTe14zZZPt6sI1W9ByOXx.jpg | [35, 10749] | 114 | en | Pretty Woman | When a millionaire wheeler-dealer enters a business contract with a Hollywood hooker Vivian Ward, he loses his heart in the bargain. | 97.953 | /hVHUfT801LQATGd26VPzhorIYza.jpg | 1990-03-23 | Pretty Woman | False | 7.5 | 7671 |
+
+
+
+Next, you will create a corresponding object collection and import the data.
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/102_mm_collections/20_create_collection.mdx b/developers/academy/js/starter_multimodal_data/102_mm_collections/20_create_collection.mdx
new file mode 100644
index 0000000000..4b2c6ad9b7
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/102_mm_collections/20_create_collection.mdx
@@ -0,0 +1,89 @@
+---
+title: Create a collection
+description: Creating Multimodal Data Collections
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/102_collection.mts';
+
+Weaviate stores data in "collections". A collection is a set of objects that share the same data structure. In our movie database, we might have a collection of movies, a collection of actors, and a collection of reviews.
+
+Here we will create a collection of movies.
+
+## Code
+
+This example creates a collection for the movie data:
+
+
+
+Each collection definition must have a name. Then, you can define additional parameters like we've done in this example.
+
+## Explain the code
+
+### Properties
+
+Properties are the object attributes that you want to store in the collection. Each property has a name and a data type.
+
+In our movie database, we have properties like `title`, `release_date` and `genre_ids`, with data types like `TEXT` (string), `DATE` (date), or `INT` (integer). It's also possible to have arrays of integers, like we have with `genre_ids`.
+
+As a multimodal object, we also have the `poster` property which is the image data, which is saved as a `BLOB` (binary large object) data type.
+
+#### Auto-schema
+
+Weaviate can automatically [infer the schema](/developers/weaviate/config-refs/schema/index.md#auto-schema) from the data. However, it's a good practice to define the properties explicitly, for better control and to avoid surprises.
+
+### Vectorizer configuration
+
+If you do not specify the vector yourself, Weaviate will use a specified vectorizer to generate vector embeddings from your data.
+
+In this code example, we specify the `multi2vec-voyageai` module. This module uses the **voyage-multimodal-3** model to generate vector embeddings from the text and image data.
+
+You can specify any number of text and image properties to be used for vectorization, and weight them differently. The weights are used to determine the relative importance of each property in the vector embedding generation process. In this example, we vectorize the `poster` property (an image) with a 90% weight and the `title` property (a string) with a 10% weight.
+
+
+
+### Generative configuration
+
+If you wish to use your collection with a generative model (e.g. a large language model), you must specify the generative module.
+
+In this code example, we specify the `cohere` module (`generative-cohere` is the full name) with default options.
+
+
+
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+### TypeScript named imports
+
+The code example makes use of named imports such as `vectorizer` and `configure`. They are defined in the `weaviate` module and are used during the collection definition.
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/102_mm_collections/30_import_data.mdx b/developers/academy/js/starter_multimodal_data/102_mm_collections/30_import_data.mdx
new file mode 100644
index 0000000000..3d274c01d0
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/102_mm_collections/30_import_data.mdx
@@ -0,0 +1,107 @@
+---
+title: Import data
+description: Importing Data into Multimodal Data Collections
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/102_collection.mts';
+
+## Code
+
+This example imports the movie data into our collection.
+
+
+
+The code:
+- Loads the source text and image data
+- Gets the collection
+- Loops through the data and:
+ - Finds corresponding image to the text
+ - Converts the image to base64
+ - Bulk inserts objects in batches of 20
+- Prints out any import errors
+
+## Explain the code
+
+### Preparation
+
+We use the native Node.js `fetch()` to load the data from the source, in this case a JSON file containing text data and a Zip file containing posters. The text data is then converted to a JSON object for easier manipulation and the images are extracted from the Zip file.
+
+Then, we create a collection object (with `client.collections.get`) so we can interact with the collection.
+
+### Iterating over data
+
+The `for` loop is used in conjunction with `Object.keys()` to iterate through the elements in our JSON file. While iterating we increment the counter variable that lets us bulk insert objects in batches.
+
+
+
+
+### Add data to the Object
+
+#### Convert data types and build the Object
+
+The data is converted from a string to the correct data types for Weaviate. For example, the `release_date` is converted to a `Date` object, and the `genre_ids` are converted to a list of integers.
+
+
+
+To save the image data as a `BLOB` (binary large object) data type, we convert the image to base64 using the helpful `toBase64FromMedia` utility that comes with the Weaviate client.
+
+
+
+After converting data to the correct format, we build the object by its properties preparing it to be inserted into Weaviate.
+
+#### Bulk insert data
+
+Then we create on object that includes the uuid generated with `generateUuid5` from Weaviate and the object containing properties we previously define, we push this object to `itemsToInsert` for them to be bulk inserted with `insertMany()` once the batch is ready.
+
+
+
+### Error handling
+
+If you have any errors in your bulk insertion, you want to know something went wrong. That way you can decide how to handle them, such as by raising an exception. In this example, we simply print out that there was an error with the import.
+
+
+
+## Where do the vectors come from?
+
+When sends the items to Weaviate, the objects are added to the collection. In our case, the movie collection.
+
+Recall that the collection has a vectorizer module, and we do not specify vectors here. So Weaviate uses the specified vectorizer to generate vector embeddings from the data.
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/102_mm_collections/index.mdx b/developers/academy/js/starter_multimodal_data/102_mm_collections/index.mdx
new file mode 100644
index 0000000000..5ad92db5a1
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/102_mm_collections/index.mdx
@@ -0,0 +1,26 @@
+---
+title: Populate the database
+description: Multimodal Data Collections Overview
+---
+
+
+
+
+
+
+
+
+## Learning objectives
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/103_mm_searches/10_multimodal.mdx b/developers/academy/js/starter_multimodal_data/103_mm_searches/10_multimodal.mdx
new file mode 100644
index 0000000000..d8c03e94a9
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/103_mm_searches/10_multimodal.mdx
@@ -0,0 +1,145 @@
+---
+title: Multimodal search
+description: Multimodal Search Methodology
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/103_searches.mts';
+
+With Weaviate, you can perform semantic searches to find similar items based on their meaning. This is done by comparing the vector embeddings of the items in the database.
+
+As we are using a multimodal model, we can search for objects based on their similarity to any of the supported modalities. Meaning that we can search for movies based on their similarity to a text or an image.
+
+## Image query
+
+### Code
+
+This example finds entries in "Movie" based on their similarity to [this image of the International Space Station](https://upload.wikimedia.org/wikipedia/commons/thumb/0/04/International_Space_Station_after_undocking_of_STS-132.jpg/440px-International_Space_Station_after_undocking_of_STS-132.jpg), and prints out the title and release year of the top 5 matches.
+
+
+ Query image
+
+
+
+
+
+
+
+### Explain the code
+
+The results are based on similarity of the vector embeddings between the query and the database object. In this case, the vectorizer module generates an embedding of the input image.
+
+The `limit` parameter here sets the maximum number of results to return.
+
+The `returnMetadata` parameter takes an array of strings to set metadata to return in the search results. The current query returns the vector distance to the query.
+
+
+
+ Example results
+
+Posters for the top 5 matches:
+
+
+
+
+
+
+Weaviate output:
+
+```text
+Interstellar 2014 157336
+Distance to query: 0.354
+
+Gravity 2013 49047
+Distance to query: 0.384
+
+Arrival 2016 329865
+Distance to query: 0.386
+
+Armageddon 1998 95
+Distance to query: 0.400
+
+Godzilla 1998 929
+Distance to query: 0.441
+```
+
+
+
+### Response object
+
+The returned object is an instance of a custom class. Its `objects` attribute is a list of search results, each object being an instance of another custom class.
+
+Each returned object will:
+- Include all properties and its UUID by default except those with blob data types.
+ - Since the `poster` property is a blob, it is not included by default.
+ - To include the `poster` property, you must specify it and the other properties to fetch in the `returnProperties` parameter.
+- Not include any other information (e.g. references, metadata, vectors.) by default.
+
+
+## Text search
+
+### Code
+
+This example finds entries in "Movie" based on their similarity to the query "red", and prints out the title and release year of the top 5 matches.
+
+
+
+### Explain the code
+
+The results are based on similarity of the vector embeddings between the query and the database object. In this case, the vectorizer module generates an embedding of the input text.
+
+The remaining parameters are the same as in the previous example.
+
+
+ Example results
+
+Posters for the top 5 matches:
+
+
+
+
+
+
+Weaviate output:
+
+```text
+Deadpool 2 2018 383498
+Distance to query: 0.670
+
+Bloodshot 2020 338762
+Distance to query: 0.677
+
+Deadpool 2016 293660
+Distance to query: 0.678
+
+300 2007 1271
+Distance to query: 0.682
+
+The Hunt for Red October 1990 1669
+Distance to query: 0.683
+```
+
+
+
+### Response object
+
+The returned object is in the same format as in the previous example.
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/103_mm_searches/20_keyword_hybrid.mdx b/developers/academy/js/starter_multimodal_data/103_mm_searches/20_keyword_hybrid.mdx
new file mode 100644
index 0000000000..a73e857202
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/103_mm_searches/20_keyword_hybrid.mdx
@@ -0,0 +1,105 @@
+---
+title: Keyword & Hybrid search
+description: Hybrid Keyword Searches in Multimodal Collections
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/103_searches.mts';
+
+You can also perform keyword (BM25) searches to find items based on their keyword similarity, or hybrid searches that combine BM25 and semantic/vector searches.
+
+## Keyword search
+
+### Code
+
+This example finds entries in "Movie" with the highest keyword search scores for the term "history", and prints out the title and release year of the top 5 matches.
+
+
+
+### Explain the code
+
+The results are based on a keyword search score using what's called the [BM25f](https://en.wikipedia.org/wiki/Okapi_BM25) algorithm.
+
+The `limit` parameter here sets the maximum number of results to return.
+
+The `returnMetadata` parameter takes an array of strings to set metadata to return in the search results. The current query returns the `score`, which is the BM25 score of the result.
+
+
+ Example results
+
+```text
+American History X 1998
+BM25 score: 2.707
+
+A Beautiful Mind 2001
+BM25 score: 1.896
+
+Legends of the Fall 1994
+BM25 score: 1.663
+
+Hacksaw Ridge 2016
+BM25 score: 1.554
+
+Night at the Museum 2006
+BM25 score: 1.529
+```
+
+
+
+
+## Hybrid search
+
+### Code
+
+This example finds entries in "Movie" with the highest hybrid search scores for the term "history", and prints out the title and release year of the top 5 matches.
+
+
+
+### Explain the code
+
+The results are based on a hybrid search score. A hybrid search blends results of BM25 and semantic/vector searches.
+
+The `limit` parameter here sets the maximum number of results to return.
+
+The `returnMetadata` parameter takes an array of strings to set metadata to return in the search results. The current query returns the `score`, which is the hybrid score of the result.
+
+
+ Example results
+
+```text
+Legends of the Fall 1994
+Hybrid score: 0.016
+
+Hacksaw Ridge 2016
+Hybrid score: 0.016
+
+A Beautiful Mind 2001
+Hybrid score: 0.015
+
+The Butterfly Effect 2004
+Hybrid score: 0.015
+
+Night at the Museum 2006
+Hybrid score: 0.012
+```
+
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/103_mm_searches/30_filters.mdx b/developers/academy/js/starter_multimodal_data/103_mm_searches/30_filters.mdx
new file mode 100644
index 0000000000..b9ad7aadd8
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/103_mm_searches/30_filters.mdx
@@ -0,0 +1,55 @@
+---
+title: Filters
+description: Filters for Multimodal Searches
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/103_searches.mts';
+
+Filters can be used to precisely refine search results. You can filter by properties as well as metadata, and you can combine multiple filters with `and` or `or` conditions to further narrow down the results.
+
+### Code
+
+This example finds entries in "Movie" based on their similarity to the query "dystopian future", only from those released after 2010. It prints out the title and release year of the top 5 matches.
+
+
+
+## Explain the code
+
+This query is identical to [that shown earlier](./10_multimodal.mdx) for search, but with the addition of a filter. The `filters` parameter makes use of the `filter` namespace to set the filter conditions. The current query filters the results to only include those with a release year after 2010.
+
+
+ Example results
+
+```text
+Dune 2021
+Distance to query: 0.199
+
+Tenet 2020
+Distance to query: 0.200
+
+Mission: Impossible - Dead Reckoning Part One 2023
+Distance to query: 0.207
+
+Onward 2020
+Distance to query: 0.214
+
+Jurassic World Dominion 2022
+Distance to query: 0.216
+```
+
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/103_mm_searches/index.mdx b/developers/academy/js/starter_multimodal_data/103_mm_searches/index.mdx
new file mode 100644
index 0000000000..a12cd26008
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/103_mm_searches/index.mdx
@@ -0,0 +1,26 @@
+---
+title: Perform searches
+description: Multimodal Searches Overview
+---
+
+
+
+
+
+
+
+
+## Learning objectives
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/104_mm_rag/10_setup.mdx b/developers/academy/js/starter_multimodal_data/104_mm_rag/10_setup.mdx
new file mode 100644
index 0000000000..ecd42c2c13
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/104_mm_rag/10_setup.mdx
@@ -0,0 +1,40 @@
+---
+title: "RAG: Overview"
+description: Setting up Multimodal RAG
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/102_collection.mts';
+
+### Motivation
+
+Retrieval augmented generation (RAG) is a way to combine the best of both worlds: the retrieval capabilities of semantic search and the generation capabilities of AI models such as large language models. This allows you to retrieve objects from a Weaviate instance and then generate outputs based on the retrieved objects.
+
+### Setup
+
+When we created a collection, we specified the `generative` parameter as shown here.
+
+
+
+This selects a generative module that will be used to generate outputs based on the retrieved objects. In this case, we're using the `cohere` module, and the `command` family of large language models.
+
+As we did before with the vectorizer module, you will require an API key from the provider of the generative module. In this case, you will need an API key from Cohere.
+
+### RAG queries
+
+RAG queries are also called 'generative' queries in Weaviate. You can access these functions through the `generate` submodule of the collection object.
+
+Each generative query works in addition to the regular search query, and will perform a RAG query on each retrieved object.
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/104_mm_rag/20_single_prompt.mdx b/developers/academy/js/starter_multimodal_data/104_mm_rag/20_single_prompt.mdx
new file mode 100644
index 0000000000..5ac9281905
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/104_mm_rag/20_single_prompt.mdx
@@ -0,0 +1,57 @@
+---
+title: "'Single prompt' generation"
+description: Single Prompt for Multimodal RAG
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/104_rag.mts';
+
+A 'single prompt' generation wil perform RAG queries on each retrieved object. This is useful when you want to transform each object separately, with the same prompt.
+
+### Code
+
+This example finds entries in "Movie" based on their similarity to [this image of the International Space Station](https://upload.wikimedia.org/wikipedia/commons/thumb/0/04/International_Space_Station_after_undocking_of_STS-132.jpg/440px-International_Space_Station_after_undocking_of_STS-132.jpg). Then, instructs the large language model to translate the title of each movie into French.
+
+Each of the results are then printed out to the console.
+
+
+
+## Explain the code
+
+You must pass on one or more properties to the `singlePrompt` parameter through braces, as we've done here with `"... {title} ..."`. This will instruct Weaviate to pass on the `title` property from each retrieved object to the large language model.
+
+
+ Example results
+
+```text
+Interstellar
+Interstellaire
+Gravity
+Gravité
+Arrival
+Arrivée
+Armageddon
+Armageddon
+Godzilla
+Godzilla
+```
+
+
+
+### Response object
+
+Each response object is similar to that from a regular search query, with an additional `generated` attribute. This attribute will contain the generated output for each object.
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/104_mm_rag/30_grouped_task.mdx b/developers/academy/js/starter_multimodal_data/104_mm_rag/30_grouped_task.mdx
new file mode 100644
index 0000000000..2bee636b35
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/104_mm_rag/30_grouped_task.mdx
@@ -0,0 +1,60 @@
+---
+title: "'Grouped task' generation"
+description: Grouped Task for Multimodal RAG
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import TSCode from '!!raw-loader!../_snippets/104_rag.mts';
+
+A 'grouped task' generation wil perform RAG queries on the set of retrieved objects. This is useful when you want to transform the set of objects as a whole, with one prompt.
+
+### Code
+
+This example finds entries in "MovieMM" based on their similarity to [this image of the International Space Station](https://upload.wikimedia.org/wikipedia/commons/thumb/0/04/International_Space_Station_after_undocking_of_STS-132.jpg/440px-International_Space_Station_after_undocking_of_STS-132.jpg). Then, instructs the large language model to find commonalities between them.
+
+Each of the results are then printed out to the console.
+
+
+
+## Explain the code
+
+For `groupedTask` queries, you simply pass on the prompt to the `groupedTask` parameter. This will instruct Weaviate to pass on the:
+- text properties from all retrieved objects, and
+- the prompt
+
+to the large language model.
+
+
+ Example results
+
+```text
+Interstellar
+Gravity
+Arrival
+Armageddon
+Godzilla
+These movies all involve space exploration, extraterrestrial beings, or catastrophic events threatening Earth. They all deal with themes of survival, human ingenuity, and the unknown mysteries of the universe.
+```
+
+
+
+### Optional parameters
+
+You can also pass on a list of properties to be used, as the `groupedProperties` parameter. This can be useful to reduce the amount of data passed on to the large language model and omit irrelevant properties.
+
+### Response object
+
+A RAG query with the `groupedTask` parameter will return a response with an additional `generated` attribute. This attribute will contain the generated output for the set of objects.
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/104_mm_rag/index.mdx b/developers/academy/js/starter_multimodal_data/104_mm_rag/index.mdx
new file mode 100644
index 0000000000..6ff1328176
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/104_mm_rag/index.mdx
@@ -0,0 +1,26 @@
+---
+title: LLMs and Weaviate (RAG)
+description: Multimodal Retrieval-Augmented Generation (RAG) Overview
+---
+
+
+
+
+
+
+
+
+## Learning objectives
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/900_next_steps.mdx b/developers/academy/js/starter_multimodal_data/900_next_steps.mdx
new file mode 100644
index 0000000000..0ae0708103
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/900_next_steps.mdx
@@ -0,0 +1,7 @@
+---
+title: Next steps
+---
+
+import IntroNextSteps from '../_snippets/intro_next_steps_js.mdx';
+
+
diff --git a/developers/academy/js/starter_multimodal_data/_snippets/101_connect.mts b/developers/academy/js/starter_multimodal_data/_snippets/101_connect.mts
new file mode 100644
index 0000000000..d6a5742e2a
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/_snippets/101_connect.mts
@@ -0,0 +1,171 @@
+// DockerInstantiation // WCDInstantiation // WCDAPIKeyInstantiation // DockerInstantiation // DockerAPIKeyInstantiation // TryFinallyCloseDemo
+import weaviate, { WeaviateClient } from "weaviate-client";
+let client: WeaviateClient;
+// END DockerInstantiation // END WCDInstantiation // END WCDAPIKeyInstantiation // END DockerInstantiation // END DockerAPIKeyInstantiation // END TryFinallyCloseDemo
+
+
+// WCDInstantiation
+
+client = await weaviate.connectToWeaviateCloud(
+ process.env.WCD_URL as string, {
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+})
+
+// END WCDInstantiation
+
+client.close()
+
+// WCDAPIKeyInstantiation
+
+client = await weaviate.connectToWeaviateCloud(
+ process.env.WCD_URL as string, {
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+ headers: {
+ 'X-VoyageAI-Api-Key': process.env.VOYAGEAI_API_KEY as string, // Replace with your inference API key
+ }
+})
+// END WCDAPIKeyInstantiation
+
+client.close()
+
+// DockerInstantiation
+
+client = await weaviate.connectToLocal()
+// END DockerInstantiation
+
+client.close()
+
+// DockerAPIKeyInstantiation
+
+client = await weaviate.connectToLocal({
+ host: '...',
+ headers: {
+ 'X-VoyageAI-Api-Key': process.env.VOYAGEAI_APIKEY as string, // Replace with your inference API key
+ }
+})
+// END DockerAPIKeyInstantiation
+
+// PollLiveness
+if (await client.isLive()) {
+ // Run operations with the client if it is live
+}
+// END PollLiveness
+
+
+// GetMeta
+console.log(await client.getMeta())
+// END GetMeta
+
+
+const outputString =
+// OutputGetMeta
+{
+ hostname: 'http://[::]:8080',
+ modules: {
+ 'backup-gcs': {
+ bucketName: 'weaviate-wcs-prod-cust-europe-west3-workloads-backups',
+ rootName: '55a78146-dae1-4609-90ce-556db01f4a61'
+ },
+ 'generative-anyscale': {
+ documentationHref: 'https://docs.anyscale.com/endpoints/overview',
+ name: 'Generative Search - Anyscale'
+ },
+ 'generative-aws': {
+ documentationHref: 'https://docs.aws.amazon.com/bedrock/latest/APIReference/welcome.html',
+ name: 'Generative Search - AWS'
+ },
+ 'generative-cohere': {
+ documentationHref: 'https://docs.cohere.com/reference/chat',
+ name: 'Generative Search - Cohere'
+ },
+ 'generative-mistral': {
+ documentationHref: 'https://docs.mistral.ai/api/',
+ name: 'Generative Search - Mistral'
+ },
+ 'generative-openai': {
+ documentationHref: 'https://platform.openai.com/docs/api-reference/completions',
+ name: 'Generative Search - OpenAI'
+ },
+ 'generative-google': {
+ documentationHref: 'https://cloud.google.com/vertex-ai/docs/generative-ai/chat/test-chat-prompts',
+ name: 'Generative Search - Google'
+ },
+ 'multi2vec-google': {
+ documentationHref: 'https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings',
+ name: 'Google Multimodal Module'
+ },
+ 'qna-openai': {
+ documentationHref: 'https://platform.openai.com/docs/api-reference/completions',
+ name: 'OpenAI Question & Answering Module'
+ },
+ 'ref2vec-centroid': {},
+ 'reranker-cohere': {
+ documentationHref: 'https://txt.cohere.com/rerank/',
+ name: 'Reranker - Cohere'
+ },
+ 'reranker-voyageai': {
+ documentationHref: 'https://docs.voyageai.com/reference/reranker-api',
+ name: 'Reranker - VoyageAI'
+ },
+ 'text2vec-aws': {
+ documentationHref: 'https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html',
+ name: 'AWS Module'
+ },
+ 'text2vec-cohere': {
+ documentationHref: 'https://docs.cohere.ai/embedding-wiki/',
+ name: 'Cohere Module'
+ },
+ 'text2vec-huggingface': {
+ documentationHref: 'https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task',
+ name: 'Hugging Face Module'
+ },
+ 'text2vec-jinaai': {
+ documentationHref: 'https://jina.ai/embeddings/',
+ name: 'JinaAI Module'
+ },
+ 'text2vec-openai': {
+ documentationHref: 'https://platform.openai.com/docs/guides/embeddings/what-are-embeddings',
+ name: 'OpenAI Module'
+ },
+ 'text2vec-google': {
+ documentationHref: 'https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings',
+ name: 'Google Module'
+ },
+ 'text2vec-voyageai': {
+ documentationHref: 'https://docs.voyageai.com/docs/embeddings',
+ name: 'VoyageAI Module'
+ }
+ },
+ version: '1.25.5'
+}
+// END OutputGetMeta
+
+
+
+client.close()
+
+client = await weaviate.connectToWeaviateCloud(
+ process.env.WCD_URL as string,
+ {
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+ headers: {
+ 'X-VoyageAI-Api-Key': process.env.VOYAGEAI_API_KEY as string, // Replace with your inference API key
+ }
+ }
+)
+
+// TryFinallyCloseDemo
+
+// Instantiate your client (not shown). e.g.:
+// client = weaviate.connectToWeaviateCloud(...) or
+// client = weaviate.connectToLocal(...)
+
+try {
+ if (await client.isLive()) {
+ // Work with the client here
+ // ...
+ }
+} finally { // This will always be executed, even if an exception is raised
+ client.close() // Close the connection & release resources
+}
+// END TryFinallyCloseDemo
diff --git a/developers/academy/js/starter_multimodal_data/_snippets/102_collection.mts b/developers/academy/js/starter_multimodal_data/_snippets/102_collection.mts
new file mode 100644
index 0000000000..a01ca6f2d0
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/_snippets/102_collection.mts
@@ -0,0 +1,210 @@
+import { promises as fs } from 'fs';
+import { join } from 'path';
+import AdmZip from 'adm-zip';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+import 'dotenv/config'
+
+// CreateMovieCollection // SubmoduleImport // BatchImportData
+import weaviate from "weaviate-client";
+// END BatchImportData // END CreateMovieCollection // END SubmoduleImport
+// CreateMovieCollection // SubmoduleImport
+import { WeaviateClient, configure, vectorizer, toBase64FromMedia } from "weaviate-client";
+// END CreateMovieCollection // END SubmoduleImport
+
+
+// BatchImportData
+import { generateUuid5 } from "weaviate-client";
+
+// END BatchImportData
+
+// BatchImportData
+let client: WeaviateClient;
+// CreateMovieCollection // END BatchImportData
+
+// END CreateMovieCollection
+
+const wcdURL = process.env.WCD_URL as string;
+const wcdApikey = process.env.WCD_API_KEY as string;
+const cohereApiKey = process.env.COHERE_API_KEY as string;
+const voyageApiKey = process.env.VOYAGEAI_API_KEY as string;
+
+
+
+// client = await weaviate.connectToWeaviateCloud(wcdURL, {
+// authCredentials: new weaviate.ApiKey(wcdApikey),
+// }
+// )
+// CreateMovieCollection
+// Instantiate your client (not shown). e.g.:
+// const requestHeaders = {'X-VoyageAI-Api-Key': process.env.VOYAGEAI_API_KEY as string,}
+// client = weaviate.connectToWeaviateCloud(..., headers: requestHeaders) or
+// client = weaviate.connectToLocal(..., headers: requestHeaders)
+
+// END CreateMovieCollection
+
+const requestHeaders = { 'X-VoyageAI-Api-Key': voyageApiKey,
+ 'X-Cohere-Api-Key': cohereApiKey
+ }
+
+
+client = await weaviate.connectToWeaviateCloud(wcdURL, {
+ authCredentials: new weaviate.ApiKey(wcdApikey),
+ headers: requestHeaders
+}
+)
+// END CreateMovieCollection
+
+// Actual instantiation
+
+client.collections.delete("Movie")
+
+// CreateMovieCollection
+await client.collections.create({
+ name: "Movie",
+ properties: [
+ { name: "title", dataType: configure.dataType.TEXT },
+ { name: "overview", dataType: configure.dataType.TEXT },
+ { name: "vote_average", dataType: configure.dataType.NUMBER },
+ { name: "genre_ids", dataType: configure.dataType.INT_ARRAY },
+ { name: "release_date", dataType: configure.dataType.DATE },
+ { name: "tmdb_id", dataType: configure.dataType.INT },
+ { name: "poster", dataType: configure.dataType.BLOB }
+ ],
+ // Define the vectorizer module
+ vectorizers: vectorizer.multi2VecVoyageAI({
+ imageFields: [{ name: "poster", weight: 0.9 }],
+ textFields: [{ name: "title", weight: 0.1 }],
+ model: "voyage-multimodal-3"
+ }),
+ // Define the generative module
+ generative: configure.generative.cohere(),
+ // END generativeDefinition // CreateMovieCollection
+})
+
+client.close()
+// END CreateMovieCollection
+
+const weaviateURL = process.env.WCD_URL as string
+const weaviateKey = process.env.WCD_API_KEY as string
+const cohereKey = process.env.COHERE_API_KEY as string
+const voyageaiKey = process.env.VOYAGEAI_API_KEY as string
+
+client = await weaviate.connectToWeaviateCloud(weaviateURL, {
+ authCredentials: new weaviate.ApiKey(weaviateKey),
+ headers: {
+ 'X-VoyageAI-Api-Key': voyageaiKey, // Replace with your inference API key
+ 'X-Cohere-Api-Key': cohereKey, // Replace with your inference API key
+ }
+})
+
+// BatchImportData
+
+// Instantiate your client (not shown). e.g.:
+// client = weaviate.connectToWeaviateCloud(...) or
+// client = weaviate.connectToLocal(...)
+
+// END BatchImportData
+
+// BatchImportData
+const dataUrl = "https://raw.githubusercontent.com/weaviate-tutorials/edu-datasets/main/movies_data_1990_2024.json"
+const textResponse = await fetch(dataUrl)
+const data = await textResponse.json()
+
+// Get current file's directory
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const imgDir = join(__dirname, "images");
+
+
+// Create directory if it doesn't exist
+await fs.mkdir(imgDir, { recursive: true });
+
+// Download images
+const postersUrl = "https://raw.githubusercontent.com/weaviate-tutorials/edu-datasets/main/movies_data_1990_2024_posters.zip";
+const postersPath = join(imgDir, "movies_data_1990_2024_posters.zip");
+
+const response = await fetch(postersUrl);
+if (!response.ok) {
+ throw new Error(`HTTP error! status: ${response.status}`);
+}
+const arrayBuffer = await response.arrayBuffer();
+const buffer = Buffer.from(arrayBuffer);
+
+// Write the zip file
+await fs.writeFile(postersPath, buffer);
+
+// Unzip the files
+const zip = new AdmZip(postersPath);
+zip.extractAllTo(imgDir, true);
+
+// Get the collection
+const movies = client.collections.get("Movie")
+
+// Set a counter and initialize Weaviate Object
+let itemsToInsert: Object[] = []
+let counter = 0;
+
+// Iterate through data
+for (const key of Object.keys(data['title'])) {
+
+ counter++;
+ if (counter % 20 == 0)
+ console.log(`Import: ${counter}`)
+ // END Iterate through data // END BatchImportData
+ // BatchImportData
+
+ let genreIds: []
+
+ // Format genre_ids and release_date
+ const parsedArray = JSON.parse(data['genre_ids'][key]);
+ genreIds = parsedArray.map((item: string) => parseInt(item, 10));
+ let releaseDate = new Date(data['release_date'][key])
+
+ const imgPath = join(imgDir, `${data['id'][key]}_poster.jpg`)
+ // Convert poster to base64
+ const posterBase64 = await toBase64FromMedia(imgPath)
+
+ // Build the object payload
+ let movieObject = {
+ title: data['title'][key],
+ overview: data['overview'][key],
+ vote_average: data['vote_average'][key],
+ genre_ids: genreIds,
+ release_date: releaseDate,
+ tmdb_id: data['id'][key],
+ poster: posterBase64
+ }
+ // Insert
+ let objectToInsert = {
+ properties: movieObject,
+ uuid: generateUuid5(data['title'][key])
+ }
+
+ // Add object to batching array
+ itemsToInsert.push(objectToInsert)
+
+ if (itemsToInsert.length == 20) {
+ try {
+ const response = await movies.data.insertMany(itemsToInsert);
+ // END Insert
+ // Handle Errors // Insert
+ if (response.hasErrors) {
+ throw new Error("Error in batch import!");
+ }
+ // END Insert // END Handle Errors
+ // Insert
+ console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
+ itemsToInsert = [];
+ } catch (error) {
+ console.error('Error importing batch:', error);
+ }
+ }
+ // END BatchImportData // END Insert
+ // BatchImportData // Iterate through data
+ // ... other operations
+}
+
+
+client.close()
+// END BatchImportData
diff --git a/developers/academy/js/starter_multimodal_data/_snippets/103_searches.mts b/developers/academy/js/starter_multimodal_data/_snippets/103_searches.mts
new file mode 100644
index 0000000000..6578521635
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/_snippets/103_searches.mts
@@ -0,0 +1,154 @@
+import 'dotenv/config'
+// START-ANY
+import weaviate, { WeaviateClient, WeaviateReturn } from "weaviate-client";
+let client: WeaviateClient;
+let response: WeaviateReturn
+// END-ANY
+
+client = await weaviate.connectToWeaviateCloud(process.env.WCD_URL as string,{
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+ headers: {
+ 'X-VoyageAI-Api-Key': process.env.VOYAGEAI_API_KEY as string, // Replace with your inference API key
+ 'X-Cohere-Api-Key': process.env.COHERE_API_KEY as string, // Replace with your inference API key
+
+ }
+ }
+)
+
+// START-ANY
+
+// Instantiate your client (not shown). e.g.:
+// const requestHeaders = {'X-VoyageAI-Api-Key': process.env.VOYAGEAI_API_KEY as string,}
+// client = weaviate.connectToWeaviateCloud(..., headers: requestHeaders) or
+// client = weaviate.connectToLocal(..., headers: requestHeaders)
+
+async function urlToBase64(imageUrl: string) {
+ const response = await fetch(imageUrl);
+ const arrayBuffer = await response.arrayBuffer();
+ const content = Buffer.from(arrayBuffer);
+ return content.toString('base64');
+}
+// END-ANY
+
+// MetadataMultimodalSearch // SinglePromptGeneration // MetadataSemanticSearch // MetadataBM25Search // MetadataHybridSearch // FilteredSemanticSearch
+
+// Get the collection
+const movies = client.collections.get("Movie")
+
+// Perform query
+const srcImgPath = "https://github.com/weaviate-tutorials/edu-datasets/blob/main/img/International_Space_Station_after_undocking_of_STS-132.jpg?raw=true"
+const queryB64 = await urlToBase64(srcImgPath)
+// END MetadataMultimodalSearch // END SinglePromptGeneration // END MetadataSemanticSearch // END MetadataBM25Search // END MetadataHybridSearch // END FilteredSemanticSearch
+
+// MetadataMultimodalSearch
+
+response = await movies.query.nearImage(queryB64, {
+ limit: 5,
+ returnMetadata: ['distance'],
+ returnProperties: ["title", "tmdb_id", "release_date", "poster"]
+ },
+)
+
+// Inspect the response
+for (let item of response.objects) {
+ // Print the title and release year (note the release date is a datetime object)
+ console.log(`${item.properties.title} - ${item.properties.release_date}`)
+ // Print the distance of the object from the query
+ console.log(`Distance to query: ${item.metadata?.distance}`)
+}
+
+client.close()
+// END MetadataMultimodalSearch
+
+console.log("\n\n")
+
+
+// MetadataSemanticSearch
+response = await movies.query.nearText("red", {
+ limit: 5,
+ returnMetadata: ['distance'],
+ returnProperties: ["title", "tmdb_id", "release_date"]
+ },
+)
+
+// Inspect the response
+for (let item of response.objects) {
+ // Print the title and release year (note the release date is a datetime object)
+ console.log(`${item.properties.title} - ${item.properties.release_date}`)
+ // Print the distance of the object from the query
+ console.log(`Distance to query: ${item.metadata?.distance}`)
+}
+
+client.close()
+// END MetadataSemanticSearch
+
+// MetadataBM25Search
+
+response = await movies.query.bm25("history", {
+ limit: 5,
+ returnMetadata: ['score'],
+},
+)
+
+// Inspect the response
+for (let item of response.objects) {
+ // Print the title and release year (note the release date is a datetime object)
+ console.log(`${item.properties.title} - ${item.properties.release_date}`)
+ // Print the distance of the object from the query
+ console.log(`BM25 score: ${item.metadata?.score}`)
+}
+
+client.close()
+// END MetadataBM25Search
+
+
+console.log("\n\n")
+
+// client.connect()
+
+// MetadataHybridSearch
+
+response = await movies.query.hybrid("history", {
+ limit: 5,
+ returnMetadata: ['score'],
+ returnProperties: ["title", "tmdb_id", "release_date"]
+},
+)
+
+// Inspect the response
+for (let item of response.objects) {
+ // Print the title and release year (note the release date is a datetime object)
+ console.log(`${item.properties.title} - ${item.properties.release_date}`)
+ // Print the hybrid score of the object from the query
+ console.log(`Hybrid score: ${item.metadata?.score}`)
+}
+
+client.close()
+// END MetadataHybridSearch
+
+
+console.log("\n\n")
+
+// client.connect()
+
+// FilteredSemanticSearch
+
+const filterTime = new Date(2010, 1, 1)
+
+response = await movies.query.nearText("dystopian future", {
+ limit: 5,
+ returnMetadata: ['distance'],
+ filters: movies.filter.byProperty("release_date").greaterThan(filterTime)
+ }
+)
+
+// Inspect the response
+for (let item of response.objects) {
+ // Print the title and release year (note the release date is a datetime object)
+ console.log(`${item.properties.title} - ${item.properties.release_date}`)
+ // Print the distance of the object from the query
+ console.log(`Distance to query: ${item.metadata?.distance}`)
+}
+
+client.close()
+// END FilteredSemanticSearch
diff --git a/developers/academy/js/starter_multimodal_data/_snippets/104_rag.mts b/developers/academy/js/starter_multimodal_data/_snippets/104_rag.mts
new file mode 100644
index 0000000000..0b353ec6cb
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/_snippets/104_rag.mts
@@ -0,0 +1,88 @@
+import 'dotenv/config'
+// START-ANY
+import weaviate, { GenerativeReturn, WeaviateClient } from "weaviate-client";
+let client: WeaviateClient;
+let response: GenerativeReturn
+// END-ANY
+
+client = await weaviate.connectToWeaviateCloud(
+ process.env.WCD_URL as string,
+ {
+ authCredentials: new weaviate.ApiKey(process.env.WCD_API_KEY as string),
+ headers: {
+ 'X-VoyageAI-Api-Key': process.env.VOYAGEAI_API_KEY as string, // Replace with your inference API key
+ }
+ }
+)
+
+// START-ANY
+
+// Instantiate your client (not shown). e.g.:
+// const requestHeaders = {'X-VoyageAI-Api-Key': process.env.VOYAGEAI_API_KEY as string,}
+// client = weaviate.connectToWeaviateCloud(..., headers: requestHeaders) or
+// client = weaviate.connectToLocal(..., headers: requestHeaders)
+
+async function urlToBase64(imageUrl: string) {
+ const response = await fetch(imageUrl);
+ const arrayBuffer = await response.arrayBuffer();
+ const content = Buffer.from(arrayBuffer);
+ return content.toString('base64');
+}
+
+// END-ANY
+
+// SinglePromptGeneration // GroupedTaskGeneration
+// Get the collection
+const movies = client.collections.get("Movie")
+
+// Perform query
+const srcImgPath = "https://github.com/weaviate-tutorials/edu-datasets/blob/main/img/International_Space_Station_after_undocking_of_STS-132.jpg?raw=true"
+const queryB64 = await urlToBase64(srcImgPath)
+// END SinglePromptGeneration // END GroupedTaskGeneration
+
+// SinglePromptGeneration
+
+response = await movies.generate.nearMedia(queryB64, "image",{
+ // highlight-start
+ singlePrompt: "Translate this into French: {title}"
+ // highlight-end
+ }, {
+ limit: 5
+})
+
+// Inspect the response
+for (let item of response.objects) {
+ console.log(item.properties.title)
+ console.log(item.generated)
+}
+
+client.close()
+// END SinglePromptGeneration
+
+
+console.log("\n\n")
+
+
+// GroupedTaskGeneration
+
+response = await movies.generate.nearMedia(queryB64, "image",{
+ // highlight-start
+ groupedTask: "What do these movies have in common?",
+ groupedProperties: ["title", "overview"] // Optional parameter; for reducing prompt length
+ // highlight-end
+ },{
+ limit: 5
+ }
+)
+
+// Inspect the response
+for (let item of response.objects) {
+ console.log('Title: ', item.properties.title) // Print the title
+}
+
+// highlight-start
+console.log(response.generated) // Print the generated text (the commonalities between them)
+// highlight-end
+
+client.close()
+// END GroupedTaskGeneration
diff --git a/developers/academy/js/starter_multimodal_data/index.md b/developers/academy/js/starter_multimodal_data/index.md
new file mode 100644
index 0000000000..77910e5500
--- /dev/null
+++ b/developers/academy/js/starter_multimodal_data/index.md
@@ -0,0 +1,24 @@
+---
+title: "101M Work with: Multimodal data"
+description: Learn to handle multimodal data in Weaviate for diverse data integrations.
+sidebar_position: 102
+---
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+import CourseUnits from '/src/components/Academy/courseUnits.jsx';
+import { courseData } from '/src/components/Academy/courseData.js'
+
+## Course overview
+
+In this project-based course, you will learn how to work with multimodal data using Weaviate and a movie dataset.
+
+You will get hands-on experience on how to store and index text and image data to be searchable together by meaning, using Weaviate's vectorization capabilities. You will learn how to search through that data using multimodal search methods, as well as filters. You will also learn how to use Weaviate's retrieval augmented generation (RAG) capabilities to generate outputs based on the retrieved objects.
+
+## Learning objectives
+
+
+
+## Units
+
+
+
diff --git a/developers/academy/js/starter_text_data/101_setup_weaviate/10_client.mdx b/developers/academy/js/starter_text_data/101_setup_weaviate/10_client.mdx
index 6100fbdec5..6dae251540 100644
--- a/developers/academy/js/starter_text_data/101_setup_weaviate/10_client.mdx
+++ b/developers/academy/js/starter_text_data/101_setup_weaviate/10_client.mdx
@@ -1,5 +1,6 @@
---
title: Weaviate JavaScript/Typescript client
+description: "Configure the Weaviate JavaScript client for efficient data access and management."
---
## Installation
diff --git a/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/10_create_wcs.mdx b/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/10_create_wcs.mdx
index d7519274f3..4ffe0e09e4 100644
--- a/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/10_create_wcs.mdx
+++ b/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/10_create_wcs.mdx
@@ -1,5 +1,6 @@
---
title: "Option 1: A cloud WCD instance"
+description: "Create a Weaviate instance on WCS for scalable, cloud-based data projects."
---
import Tabs from '@theme/Tabs';
diff --git a/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/20_create_docker.mdx b/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/20_create_docker.mdx
index f50debcd5e..9ff2ee84d6 100644
--- a/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/20_create_docker.mdx
+++ b/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/20_create_docker.mdx
@@ -1,5 +1,6 @@
---
title: "Option 2: A local Docker instance"
+description: "Set up Weaviate with Docker for a quick and customizable local deployment."
---
import Tabs from '@theme/Tabs';
@@ -37,12 +38,10 @@ services:
- 50051:50051
restart: on-failure:0
environment:
- OPENAI_APIKEY: $OPENAI_APIKEY
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'none'
- ENABLE_MODULES: 'text2vec-cohere,text2vec-huggingface,text2vec-openai,generative-openai,generative-cohere'
+ ENABLE_API_BASED_MODULES: 'true'
BACKUP_FILESYSTEM_PATH: '/var/lib/weaviate/backups'
CLUSTER_HOSTNAME: 'node1'
...
diff --git a/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/index.mdx b/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/index.mdx
index 03f6dce85d..53c348780e 100644
--- a/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/index.mdx
+++ b/developers/academy/js/starter_text_data/101_setup_weaviate/20_create_instance/index.mdx
@@ -1,5 +1,6 @@
---
title: Create a Weaviate instance
+description: "Create a new Weaviate instance tailored for text data applications."
---
For this unit, you can choose to create a Weaviate Cloud (WCD) instance or a local Docker instance.
diff --git a/developers/academy/js/starter_text_data/101_setup_weaviate/index.mdx b/developers/academy/js/starter_text_data/101_setup_weaviate/index.mdx
index b8a67e42a5..95c249881b 100644
--- a/developers/academy/js/starter_text_data/101_setup_weaviate/index.mdx
+++ b/developers/academy/js/starter_text_data/101_setup_weaviate/index.mdx
@@ -1,10 +1,9 @@
---
title: Set up Weaviate
+description: "Learn to set up Weaviate for text data projects, including environment setup and configuration."
---
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-## Overview
-
:::warning TODO
diff --git a/developers/academy/py/zero_to_mvp/index.md b/developers/academy/py/zero_to_mvp/index.md
index 04b9c732d9..8d7839a5dd 100644
--- a/developers/academy/py/zero_to_mvp/index.md
+++ b/developers/academy/py/zero_to_mvp/index.md
@@ -1,5 +1,6 @@
---
title: (v3) Zero to MVP
+description: Build a Weaviate MVP from scratch, covering setup, data queries, and schema.
---
import LearningGoals from '/src/components/Academy/learningGoals.jsx';
diff --git a/developers/academy/py/zero_to_mvp/setup.mdx b/developers/academy/py/zero_to_mvp/setup.mdx
index 8936e07cfa..635fe18dff 100644
--- a/developers/academy/py/zero_to_mvp/setup.mdx
+++ b/developers/academy/py/zero_to_mvp/setup.mdx
@@ -1,5 +1,6 @@
---
title: P3_101A Weaviate Academy Preparation
+description: Set up Weaviate from scratch and start building your first project.
sidebar_position: 101.5
---
diff --git a/developers/academy/theory/101_hello_weaviate/10_intro_weaviate.mdx b/developers/academy/theory/101_hello_weaviate/10_intro_weaviate.mdx
index 65035ae2d4..f1f745efde 100644
--- a/developers/academy/theory/101_hello_weaviate/10_intro_weaviate.mdx
+++ b/developers/academy/theory/101_hello_weaviate/10_intro_weaviate.mdx
@@ -1,5 +1,6 @@
---
title: Introduction to Weaviate
+description: Introduction to Weaviate Theory
---
## What is Weaviate?
diff --git a/developers/academy/theory/101_hello_weaviate/15_overview_vectors.mdx b/developers/academy/theory/101_hello_weaviate/15_overview_vectors.mdx
index b23b404860..a481262728 100644
--- a/developers/academy/theory/101_hello_weaviate/15_overview_vectors.mdx
+++ b/developers/academy/theory/101_hello_weaviate/15_overview_vectors.mdx
@@ -1,5 +1,6 @@
---
title: Vectors - An overview
+description: Vector Concepts in Weaviate Theory
---
## What is a vector?
@@ -9,7 +10,7 @@ import ReactPlayer from 'react-player/lazy'
-We've covered that Weaviate is a vector database, and that a vector search is similarity-based. But, what is a vector?
+We've covered that Weaviate is a [vector database](/blog/what-is-a-vector-database), and that a vector search is similarity-based. But, what is a vector?
A vector in this context is just a series of numbers like `[1, 0]` or `[0.513, 0.155, 0.983, ..., 0.001, 0.932]`. Vectors like these are used to capture meaning.
diff --git a/developers/academy/theory/101_hello_weaviate/20_examples_1.mdx b/developers/academy/theory/101_hello_weaviate/20_examples_1.mdx
index f14cfe0b5f..92a26378e3 100644
--- a/developers/academy/theory/101_hello_weaviate/20_examples_1.mdx
+++ b/developers/academy/theory/101_hello_weaviate/20_examples_1.mdx
@@ -1,5 +1,6 @@
---
title: Examples 1 - Queries
+description: Hello Weaviate Theory - Examples Part 1
---
## Vectors in action
diff --git a/developers/academy/theory/101_hello_weaviate/25_examples_2.mdx b/developers/academy/theory/101_hello_weaviate/25_examples_2.mdx
index c232c1b656..50c39d899f 100644
--- a/developers/academy/theory/101_hello_weaviate/25_examples_2.mdx
+++ b/developers/academy/theory/101_hello_weaviate/25_examples_2.mdx
@@ -1,5 +1,6 @@
---
title: Examples 2 - More than search
+description: Hello Weaviate Theory - Examples Part 2
sidebar_position: 25
---
diff --git a/developers/academy/theory/101_hello_weaviate/index.mdx b/developers/academy/theory/101_hello_weaviate/index.mdx
index def1827785..8015e0a6fe 100644
--- a/developers/academy/theory/101_hello_weaviate/index.mdx
+++ b/developers/academy/theory/101_hello_weaviate/index.mdx
@@ -1,5 +1,6 @@
---
title: 101 Hello, Weaviate
+description: A basic introduction to Weaviate and its data management capabilities.
sidebar_position: 101 # Like a subject number (e.g. CS101)
---
diff --git a/developers/academy/theory/150_search_types/_index.md b/developers/academy/theory/150_search_types/_index.md
new file mode 100644
index 0000000000..59fc513592
--- /dev/null
+++ b/developers/academy/theory/150_search_types/_index.md
@@ -0,0 +1,27 @@
+---
+title: 150 Search Types
+description: An introduction to different search types in Weaviate
+sidebar_position: 150 # Like a subject number (e.g. CS101)
+---
+
+## Unit overview
+
+### Prerequisites
+
+- None
+
+## Learning objectives
+
+import LearningGoalsExp from '/src/components/Academy/learningGoalsExp.mdx';
+
+
+
+import LearningGoals from '/src/components/Academy/learningGoals.jsx';
+
+
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/academy/theory/150_search_types/_notes.md b/developers/academy/theory/150_search_types/_notes.md
new file mode 100644
index 0000000000..2528b659ac
--- /dev/null
+++ b/developers/academy/theory/150_search_types/_notes.md
@@ -0,0 +1,52 @@
+### Vector embeddings
+
+Vector embeddings are numerical representations of objects in a high-dimensional space. These vectors are generated by a model trained to represent the objects in a way that captures their semantic meaning.
+
+A vector embedding may look like:
+
+```json
+[0, 1, 0, 0, 1]
+```
+
+or like:
+
+```json
+[0.158, 0.011, 0.840, 0.577, 0.897, ..., 0.144]
+```
+
+A vector embedding may be typically between 300 and 2048 dimensions, although the exact number varies depending on the model. The longer the vector, the more information it can capture about the object. On the other hand, they require more resources to store, index and search.
+
+Vector embeddings that are produced by the same model are said to be "compatible", or that they are "in the same vector space".
+
+:::tip Analog: Vector embeddings as language
+An intuitive way to think of vector embeddings is as a language. Just as two people must speak the same language to communicate, two vectors must be in the same vector space to be compared. If two vectors are not in the same vector space, their values are meaningless when compared, even if they are the same length.
+:::
+
+## Vector distance
+
+Vector distance indicates how close, or far apart, two vectors are in high-dimensional space. This is a measure of the object's "semantic" similarity to the query, based on their vector embeddings.
+
+In a simple example, consider colors "SkyBlue", "LightSteelBlue", and "DarkOrange". These colors can be represented as vectors in a 3D space, with the RGB values as the vector components.
+
+| Color | Vector (R,G,B) |
+|----------------|-----------------|
+| SkyBlue | (135, 206, 235) |
+| LightSteelBlue | (176, 196, 222) |
+| DarkOrange | (255, 140, 0) |
+
+The vectors for "SkyBlue" and "LightSteelBlue" are much closer to each other than either is to "DarkOrange", reflecting their similarity as light blue colors versus an orange color.
+
+If you search a vector database containing vectors for "SkyBlue" and "DarkOrange" with a query vector for "LightSteelBlue", the search would return "SkyBlue" as the closest match.
+
+Vector search for far more complex objects, such as text, images, or audio, is based on the same principle. The vectors are generated by a model trained to represent the objects in a high-dimensional space, where the distance between vectors reflects the similarity between the objects.
+
+### Distance and search quality
+
+All compatible vectors are similar to some degree search will have some "top" search results, even if the query is not similar to any objects in the dataset.
+
+If you search a vector database containing vectors for colors "Red", "Crimson" and "LightCoral" with a query vector for "SkyBlue", the search will still return a result (e.g. "Red"), even if it is not semantically similar to the query. The search is simply returning the closest match, even if it is not a good match in the absolute sense.
+
+To ensure that the search results are meaningful, consider the following strategies:
+
+- **Use a threshold**: Set a minimum similarity score for the results. This will exclude results that are not similar enough to the query.
+- **Apply filters**: Use [filters](../filtering.md) to exclude results based on other criteria, such as metadata or properties.
diff --git a/developers/academy/theory/index.md b/developers/academy/theory/index.md
index 168d94d62f..2ba11fa238 100644
--- a/developers/academy/theory/index.md
+++ b/developers/academy/theory/index.md
@@ -1,5 +1,6 @@
---
title: Hello, Weaviate
+description: Explore Weaviate theory, from data structure basics to advanced concepts.
---
import LearningGoals from '/src/components/Academy/learningGoals.jsx';
diff --git a/developers/contributor-guide/getting-started/commit-guidelines.md b/developers/contributor-guide/getting-started/commit-guidelines.md
index a07d2ba6b9..2ea9bd2b11 100644
--- a/developers/contributor-guide/getting-started/commit-guidelines.md
+++ b/developers/contributor-guide/getting-started/commit-guidelines.md
@@ -1,5 +1,6 @@
---
title: Commit Guidelines
+description: Commit Guidelines for Contributing to Weaviate
sidebar_position: 7
image: og/contributor-guide/getting-started.jpg
# tags: ['contributor-guide']
diff --git a/developers/contributor-guide/getting-started/git-and-github.md b/developers/contributor-guide/getting-started/git-and-github.md
index b1dc8e940a..0bdd9ec34b 100644
--- a/developers/contributor-guide/getting-started/git-and-github.md
+++ b/developers/contributor-guide/getting-started/git-and-github.md
@@ -1,5 +1,6 @@
---
title: Git and GitHub guide
+description: Git and GitHub Guide for Weaviate
sidebar_position: 6
image: og/contributor-guide/getting-started.jpg
# tags: ['contributor-guide']
diff --git a/developers/contributor-guide/getting-started/improving-docs.md b/developers/contributor-guide/getting-started/improving-docs.md
index 2d3a42a9dc..c06d2b1ff1 100644
--- a/developers/contributor-guide/getting-started/improving-docs.md
+++ b/developers/contributor-guide/getting-started/improving-docs.md
@@ -1,5 +1,6 @@
---
title: Improving Documentation
+description: Improving Documentation for Weaviate
sidebar_position: 4
image: og/contributor-guide/getting-started.jpg
# tags: ['contributor-guide']
diff --git a/developers/contributor-guide/getting-started/index.md b/developers/contributor-guide/getting-started/index.md
index f979192cac..9f3e93f364 100644
--- a/developers/contributor-guide/getting-started/index.md
+++ b/developers/contributor-guide/getting-started/index.md
@@ -1,5 +1,6 @@
---
title: Getting Started
+description: Getting Started with Contributing to Weaviate
sidebar_position: 0
image: og/contributor-guide/getting-started.jpg
# tags: ['contributor-guide']
diff --git a/developers/contributor-guide/weaviate-core/parsing-cross-refs.md b/developers/contributor-guide/weaviate-core/parsing-cross-refs.md
index a06bc6c5a2..57c18de754 100644
--- a/developers/contributor-guide/weaviate-core/parsing-cross-refs.md
+++ b/developers/contributor-guide/weaviate-core/parsing-cross-refs.md
@@ -1,12 +1,11 @@
---
title: Parsing Objects & Resolving References
+description: Guide to parsing cross-references in Weaviate core for data linking.
sidebar_position: 5
image: og/contributor-guide/weaviate-core.jpg
# tags: ['contributor-guide']
---
-## Overview
-
Objects are parsed twice:
* First, closest to disk, immediately after reading-in the byte blob, all
@@ -75,4 +74,4 @@ import ContributorGuideMoreResources from '/_includes/more-resources-contributor
import DocsFeedback from '/_includes/docs-feedback.mdx';
-
\ No newline at end of file
+
diff --git a/developers/integrations/data-platforms/airbyte/index.md b/developers/integrations/data-platforms/airbyte/index.md
new file mode 100644
index 0000000000..6821c514d0
--- /dev/null
+++ b/developers/integrations/data-platforms/airbyte/index.md
@@ -0,0 +1,22 @@
+---
+title: Airbyte
+sidebar_position: 1
+---
+[Airbyte](https://airbyte.com/) is an open-source data integration engine that helps you consolidate your data in your data warehouses, lakes and databases. You can use Airbyte to ingest data into Weaviate.
+
+
+## Airbyte and Weaviate
+
+Weaviate is a supported [destination connector](https://airbyte.com/connectors/weaviate) in Airbyte. You can set up a source connector to extract data and import it into Weaviate using Airbyte.
+
+## Our Resources
+The resources are broken into two categories:
+[**Hands on Learning**](#hands-on-learning): Build your technical understanding with end-to-end tutorials.
+
+### Hands on Learning
+
+| Topic | Description | Resource |
+| --- | --- | --- |
+| Unleash to Weaviate | Load data from Unleash into Weaviate | [Tutorial](https://airbyte.com/how-to-sync/unleash-to-weaviate) |
+| Airtable to Weaviate | Sync data from Airtable into Weaviate | [Tutorial](https://airbyte.com/how-to-sync/airtable-to-weaviate) |
+| Monday to Weaviate | Load your data in Monday into Weaviate within minutes. | [Tutorial](https://airbyte.com/how-to-sync/monday-to-weaviate) |
diff --git a/developers/integrations/data-platforms/astronomer/index.md b/developers/integrations/data-platforms/astronomer/index.md
new file mode 100644
index 0000000000..1b5521cb73
--- /dev/null
+++ b/developers/integrations/data-platforms/astronomer/index.md
@@ -0,0 +1,23 @@
+---
+title: Astronomer
+sidebar_position: 1
+---
+[Astronomer’s Astro](https://www.astronomer.io/) is a fully managed platform built on top of Apache Airflow. It simplifies the management of Airflow at scale and data ingestion into Weaviate.
+
+
+## Astronomer and Weaviate
+The [Weaviate Airflow provider](https://www.astronomer.io/docs/learn/airflow-weaviate) offers modules to easily integrate Weaviate with Airflow.
+
+Create and run a DAG to ingest data into Weaviate.
+
+## Our Resources
+The resources are broken into two categories:
+
+[**Hands on Learning**](#hands-on-learning): Build your technical understanding with end-to-end tutorials.
+
+### Hands on Learning
+
+| Topic | Description | Resource |
+| --- | --- | --- |
+| Orchestrate Weaviate operations with Apache Airflow | Use Airflow to ingest movie descriptions into Weaviate and query the collection. | [Tutorial](https://www.astronomer.io/docs/learn/airflow-weaviate) |
+
\ No newline at end of file
diff --git a/developers/integrations/data-platforms/ibm/index.md b/developers/integrations/data-platforms/ibm/index.md
new file mode 100644
index 0000000000..71f0b19b51
--- /dev/null
+++ b/developers/integrations/data-platforms/ibm/index.md
@@ -0,0 +1,20 @@
+---
+title: IBM
+sidebar_position: 5
+image: og/integrations/home.jpg
+---
+
+[IBM](https://www.ibm.com/us-en) offers a variety of solutions for building generative AI applications and other use cases.
+
+## IBM and Weaviate
+[Docling](https://github.com/DS4SD/docling) is an open-source project built by the IBM Deep Search team. It enables developers to parse and export documents and ingest it into a Weaviate collection.
+
+## Our Resources
+[**Hands on Learning**](#hands-on-learning): Build your technical understanding with end-to-end tutorials.
+
+
+### Hands on Learning
+
+| Topic | Description | Resource |
+| --- | --- | --- |
+| Performing RAG over PDFs with Weaviate and Docling | This notebook shows you how to perform RAG over PDF documents parsed by Docling | [Notebook](https://github.com/weaviate/recipes/blob/main/integrations/data-platforms/ibm/docling/rag_over_pdfs_docling_weaviate.ipynb) |
\ No newline at end of file
diff --git a/developers/integrations/data-platforms/index.md b/developers/integrations/data-platforms/index.md
index b48d6b6b57..1ebd955d04 100644
--- a/developers/integrations/data-platforms/index.md
+++ b/developers/integrations/data-platforms/index.md
@@ -7,9 +7,12 @@ image: og/integrations/home.jpg
Data Platforms offer robust solutions for managing, processing, and analyzing large volumes of data. These platforms provide tools and services that facilitate seamless data ingestion directly into Weaviate.
Learn about how Weaviate integrates with these solutions:
+* [Airbyte](/developers/integrations/data-platforms/airbyte/)
* [Aryn](/developers/integrations/data-platforms/aryn/)
+* [Astronomer](/developers/integrations/data-platforms/astronomer/)
* [Confluent Cloud](/developers/integrations/data-platforms/confluent-cloud)
* [Context Data](/developers/integrations/data-platforms/context-data/)
* [Databricks](/developers/integrations/data-platforms/databricks/)
* [Firecrawl](/developers/integrations/data-platforms/firecrawl/)
+* [IBM](/developers/integrations/data-platforms/ibm/)
* [Unstructured](/developers/integrations/data-platforms/unstructured)
diff --git a/developers/integrations/index.md b/developers/integrations/index.md
index a8ad331e37..296498981c 100644
--- a/developers/integrations/index.md
+++ b/developers/integrations/index.md
@@ -32,7 +32,7 @@ The ecosystem is divided into these categories:
|------------------|-----------|
| Cloud Hyperscalers | [AWS](/developers/integrations/cloud-hyperscalers/aws), [Google](/developers/integrations/cloud-hyperscalers/google)|
| Compute Infrastructure | [Modal](/developers/integrations/compute-infrastructure/modal), [Replicate](/developers/integrations/compute-infrastructure/replicate) |
-| Data Platforms |[Aryn](/developers/integrations/data-platforms/aryn/), [Confluent Cloud](/developers/integrations/data-platforms/confluent-cloud), [Context Data](/developers/integrations/data-platforms/context-data/), [Databricks](/developers/integrations/data-platforms/databricks/), [Firecrawl](/developers/integrations/data-platforms/firecrawl), [Unstructured](/developers/integrations/data-platforms/unstructured) |
+| Data Platforms |[Airbyte](/developers/integrations/data-platforms/airbyte), [Aryn](/developers/integrations/data-platforms/aryn/), [Confluent Cloud](/developers/integrations/data-platforms/confluent-cloud), [Astronomer](/developers/integrations/data-platforms/astronomer), [Context Data](/developers/integrations/data-platforms/context-data/), [Databricks](/developers/integrations/data-platforms/databricks/), [Firecrawl](/developers/integrations/data-platforms/firecrawl), [IBM](/developers/integrations/data-platforms/ibm/), [Unstructured](/developers/integrations/data-platforms/unstructured) |
| LLM Frameworks | [Composio](/developers/integrations/llm-frameworks/composio/), [DSPy](/developers/integrations/llm-frameworks/dspy/), [Haystack](/developers/integrations/llm-frameworks/haystack/), [LangChain](/developers/integrations/llm-frameworks/langchain/), [LlamaIndex](/developers/integrations/llm-frameworks/llamaindex/), [Semantic Kernel](/developers/integrations/llm-frameworks/semantic-kernel/) |
| Operations | [Arize](/developers/integrations/operations/arize/), [Langtrace](/developers/integrations/operations/langtrace/), [LangWatch](/developers/integrations/operations/langwatch/), [Nomic](/developers/integrations/operations/nomic/), [Ragas](/developers/integrations/operations/ragas/), [Weights & Biases](/developers/integrations/operations/wandb/) |
diff --git a/developers/wcs/cluster-status.mdx b/developers/wcs/cluster-status.mdx
index 69cf675fa1..7bfb17b782 100644
--- a/developers/wcs/cluster-status.mdx
+++ b/developers/wcs/cluster-status.mdx
@@ -34,6 +34,19 @@ import APIOutputs from '/_includes/rest/node-endpoint-info.mdx';
+## Enabled modules
+
+Each Weaviate instance has a set of enabled modules. This list will vary depending on the instance, due to the Weaviate version and WCD policies.
+
+To see the list of enabled modules on a WCD instance, go to the WCD console, click on `Clusters`, select the cluster you want to check, and scroll to the `modules active` section.
+
+import ClusterStatusModules from '/developers/wcs/img/cluster-status.modules.png';
+
+
+
+ Available modules on your WCD instance
+
+
## Support
import SupportAndTrouble from '/_includes/wcs/support-and-troubleshoot.mdx';
diff --git a/developers/wcs/embeddings/_category_.json b/developers/wcs/embeddings/_category_.json
new file mode 100644
index 0000000000..b990d12e1b
--- /dev/null
+++ b/developers/wcs/embeddings/_category_.json
@@ -0,0 +1,4 @@
+{
+ "label": "Weaviate Embeddings",
+ "position": 80
+}
diff --git a/developers/wcs/embeddings/index.md b/developers/wcs/embeddings/index.md
new file mode 100644
index 0000000000..c87e5334c9
--- /dev/null
+++ b/developers/wcs/embeddings/index.md
@@ -0,0 +1,293 @@
+---
+title: Weaviate Embeddings
+sidebar_position: 200
+image: og/wcs/user_guides.jpg
+---
+
+Weaviate Embeddings provides secure, scalable embedding generation as a fully managed service.
+
+Weaviate Embeddings integrates with Weaviate Cloud instances to generate, store, and search embeddings without managing infrastructure.
+
+```mermaid
+%%{init: {
+ 'theme': 'base',
+ 'themeVariables': {
+ 'primaryColor': '#4a5568',
+ 'primaryTextColor': '#2d3748',
+ 'primaryBorderColor': '#718096',
+ 'lineColor': '#718096',
+ 'secondaryColor': '#f7fafc',
+ 'tertiaryColor': '#edf2f7',
+ 'fontFamily': 'Inter, system-ui, sans-serif',
+ 'fontSize': '14px',
+ 'lineHeight': '1.4',
+ 'nodeBorder': '1px',
+ 'mainBkg': '#ffffff',
+ 'clusterBkg': '#f8fafc'
+ }
+}}%%
+
+flowchart LR
+ %% Style definitions
+ classDef systemBox fill:#f8fafc,stroke:#3182ce,stroke-width:1.5px,color:#2d3748,font-weight:bold
+ classDef weaviateBox fill:#f8fafc,stroke:gray,stroke-width:0.5px,color:#2d3748,font-weight:bold
+ classDef cloudBox fill:white,stroke:#48bb78,stroke-width:2px,color:#553c9a,font-weight:bold
+ classDef providerBox fill:#f8fafc,stroke:gray,stroke-width:0.5px,color:#2d3748,font-weight:bold
+ classDef component fill:white,stroke:#a0aec0,stroke-width:1px,color:#2d3748
+ classDef edgeLabel fill:white,stroke:#e2e8f0,stroke-width:1px,color:#4a5568
+
+ %% Weaviate Cloud container
+ subgraph cloud["☁️ Weaviate Cloud"]
+ %% Weaviate Embeddings section
+ subgraph provider["Weaviate\nEmbeddings"]
+ inference["🤖 Inference\nAPI"]
+ end
+
+ %% Weaviate section
+ subgraph weaviate["Weaviate Cloud instance"]
+ vectorizer["🔌 Weaviate Embeddings\nIntegration"]
+ core["💾 Data &\nvector store"]
+ end
+ end
+
+ %% User System
+ subgraph user["🖥️ User System"]
+ data["📄 Data"]
+ end
+
+ %% Connections with curved edges
+ data --->|"1. Insert\nobjects"| core
+ core --->|"2. Request\nvector"| vectorizer
+ vectorizer --->|"3. Request\nvector"| inference
+ inference --->|"4. Vector"| vectorizer
+ vectorizer --->|"5. Vector"| core
+
+ %% Apply styles
+ class user systemBox
+ class weaviate weaviateBox
+ class cloud cloudBox
+ class provider providerBox
+ class data,core,vectorizer,inference component
+
+ %% Linkstyle for curved edges
+ linkStyle default stroke:#718096,stroke-width:3px,fill:none,background-color:white
+```
+
+With Weaviate Embeddings, you can generate embeddings for your data and queries directly from a Weaviate Cloud database instance.
+
+This means you can perform semantic, vector and hybrid searches without the need to externally generate vector embeddings, or manage additional model providers.
+
+Weaviate Embeddings is fully integrated with Weaviate Cloud, so you can manage your data and embeddings in one place.
+
+:::info Weaviate Embeddings is in technical preview
+Weaviate Embeddings is in technical preview. This means that the service is still in development and may have limited functionality.
+
+
+During the technical preview, you can use Weaviate Embeddings for free. However, the service may be subject to change.
+
+
+To try out Weaviate Embeddings, please [contact us](https://events.weaviate.io/embeddings-preview) to request access.
+:::
+
+### Key Features
+
+Weaviate Embeddings offers a fully managed service for embedding generation that is integrated with Weaviate Cloud instances.
+
+- **Single authentication**: Your Weaviate Cloud credentials are used for authorization and access to Weaviate Embeddings.
+- **Unified billing**: Your billing and usage can be managed in one place through Weaviate Cloud.
+- **Model selection**: Choose from our hand-picked selection of embedding models to generate embeddings that suit your use case.
+
+### Availability
+
+Weaviate Embeddings is a part of Weaviate Cloud, and available for Weaviate Cloud instances. It is currently not available for open-source Weaviate users.
+
+## Service Details
+
+### Models
+
+The following models are available for use with Weaviate Embeddings:
+
+- `arctic-embed-m-v1.5` (default)
+ - A 109M parameter, 768-dimensional model for enterprise retrieval tasks in English.
+ - Trained with Matryoshka Representation Learning to allow vector truncation with minimal loss.
+ - Quantization-friendly: Using scalar quantization and 256 dimensions provides 99% of unquantized, full-precision performance.
+ - Read more at the [Snowflake blog](https://www.snowflake.com/engineering-blog/arctic-embed-m-v1-5-enterprise-retrieval/), and the Hugging Face [model card](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5)
+ - Allowable `dimensions`: 768 (default), 256
+
+Additional models will be added in the future.
+
+:::info Input truncation
+Currently, input exceeding the model's context windows is truncated from the right (i.e. the end of the input).
+:::
+
+### Parameters
+
+- `model` (optional): The name of the model to use for embedding generation.
+- `dimensions` (optional): The number of dimensions to use for the generated embeddings. Only available for models that support Matryoshka Representation Learning.
+- `base_url` (optional): The base URL for the Weaviate Embeddings service. (Not required in most cases.)
+
+### Rate Limits
+
+Weaviate Embeddings does not impose rate limits on requests.
+
+### Pricing and Billing
+
+During the technical preview, Weaviate Embeddings is free to use.
+
+Pricing and billing details will be provided in the future.
+
+### Data Privacy
+
+Weaviate Embeddings is a stateless service that does not store any data.
+
+The data provided to Weaviate Embeddings is used solely for the purpose of generating embeddings.
+
+We do not store or use your data for any other purpose, including training or model improvement.
+
+#### Service and Data Location
+
+Weaviate Embeddings makes use of infrastructure located in the United States. Note that by using Weaviate Embeddings, you are agreeing to have your data transferred to the United States for processing.
+
+We may expand the service to other regions in the future.
+
+### Administration
+
+Weaviate Embeddings can be enabled from the Weaviate Cloud console, at the organization level.
+
+
+
+Once enabled, all clusters within the organization can access Weaviate Embeddings.
+
+Authentication is managed through your existing Weaviate Cloud API key.
+
+## Get Started
+
+### Prerequisites
+
+To use Weaviate Embeddings, you need a Weaviate Cloud account, and a Weaviate Cloud Serverless instance. If you do not have an account, you can sign up for free at the [Weaviate Cloud console](https://console.weaviate.cloud/).
+
+To use Weaviate Embeddings, you need:
+
+- A Weaviate Cloud instance running at least Weaviate `1.27.6`
+- A Weaviate client library that supports Weaviate Embeddings:
+ - Python client version `4.9.5` or higher
+ - JavaScript/TypeScript client version `3.2.5` or higher
+ - Go/Java clients are not yet officially supported; you must pass the `X-Weaviate-Api-Key` and `X-Weaviate-Cluster-Url` headers manually upon instantiation as shown below.
+
+### Usage
+
+To use Weaviate Embeddings, log into the Weaviate Cloud console, and enable the Weaviate Embeddings service for your organization.
+
+
+
+Once the service is enabled, you can use Weaviate Embeddings to generate embeddings for your data and queries.
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyConnect from '!!raw-loader!/developers/weaviate/model-providers/_includes/provider.connect.weaviate.py';
+import TSConnect from '!!raw-loader!/developers/weaviate/model-providers/_includes/provider.connect.weaviate.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect-weaviate-embeddings/main.go';
+import JavaConnect from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ConnectWeaviateEmbeddings.java';
+import PyCode from '!!raw-loader!/developers/weaviate/model-providers/_includes/provider.vectorizer.py';
+import TSCode from '!!raw-loader!/developers/weaviate/model-providers/_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageWeaviateTextEmbeddings.java';
+
+### Connect to Weaviate
+
+Your Weaviate Cloud credentials will be used to authorize your Weaviate Cloud instance's access for Weaviate Embeddings.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Configure the vectorizer
+
+[Configure a Weaviate index](/developers/weaviate/manage-data/collections#specify-a-vectorizer) as follows to use a Weaviate Embeddings model:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Once your collection is configured, you can start using Weaviate Embeddings to generate embeddings for your data and queries.
+
+Refer to the [Model Provider Integrations page](/developers/weaviate/model-providers/weaviate/embeddings) for further examples, on:
+- [How to use custom parameters](/developers/weaviate/model-providers/weaviate/embeddings#vectorizer-parameters)
+- [Import data](/developers/weaviate/model-providers/weaviate/embeddings#data-import)
+- [Perform searches](/developers/weaviate/model-providers/weaviate/embeddings#searches)
+
+## Additional Resources
+
+- [Model provider integrations: Weaviate Embeddings](/developers/weaviate/model-providers/weaviate/embeddings)
diff --git a/developers/wcs/img/cluster-status.modules.png b/developers/wcs/img/cluster-status.modules.png
new file mode 100644
index 0000000000..fd07c7fcb5
Binary files /dev/null and b/developers/wcs/img/cluster-status.modules.png differ
diff --git a/developers/wcs/img/enable_weaviate_embeddings.png b/developers/wcs/img/enable_weaviate_embeddings.png
new file mode 100644
index 0000000000..74796a3f08
Binary files /dev/null and b/developers/wcs/img/enable_weaviate_embeddings.png differ
diff --git a/developers/wcs/index.mdx b/developers/wcs/index.mdx
index b28a5a8464..150433b5ee 100644
--- a/developers/wcs/index.mdx
+++ b/developers/wcs/index.mdx
@@ -1,5 +1,6 @@
---
title: Weaviate Cloud
+description: Weaviate Cloud Services Overview
sidebar_position: 0
image: og/wcs/overview.jpg
---
diff --git a/developers/wcs/sandbox.mdx b/developers/wcs/sandbox.mdx
index 6f7f0df687..ef5a025b82 100644
--- a/developers/wcs/sandbox.mdx
+++ b/developers/wcs/sandbox.mdx
@@ -81,7 +81,7 @@ To export your collection definitions, follow these steps:
text={PyCodeV3}
startMarker="# START ReadAllCollections"
endMarker="# END ReadAllCollections"
- language="py"
+ language="pyv3"
/>
@@ -90,7 +90,7 @@ To export your collection definitions, follow these steps:
text={TSCode}
startMarker="// START ReadAllCollections"
endMarker="// END ReadAllCollections"
- language="ts"
+ language="tsv2"
/>
@@ -157,7 +157,7 @@ Refer to the exported schema to recreate the property definitions.
text={PyCodeV3}
startMarker="# START CreateCollectionWithProperties"
endMarker="# END CreateCollectionWithProperties"
- language="py"
+ language="pyv3"
/>
@@ -166,7 +166,7 @@ Refer to the exported schema to recreate the property definitions.
text={TSCode}
startMarker="// START CreateCollectionWithProperties"
endMarker="// END CreateCollectionWithProperties"
- language="ts"
+ language="tsv2"
/>
@@ -189,4 +189,4 @@ Sandboxes can be extended three times. Each extension is for an additional 14 da
import SupportAndTrouble from '/_includes/wcs/support-and-troubleshoot.mdx';
-
\ No newline at end of file
+
diff --git a/developers/weaviate/api/graphql/additional-operators.md b/developers/weaviate/api/graphql/additional-operators.md
index 700295fd45..7c6c8aa352 100644
--- a/developers/weaviate/api/graphql/additional-operators.md
+++ b/developers/weaviate/api/graphql/additional-operators.md
@@ -179,7 +179,7 @@ Sample client code:
text={AutocutPyCodeV3}
startMarker="# START Autocut Python"
endMarker="# END Autocut Python"
- language="py"
+ language="pyv3"
/>
@@ -188,7 +188,7 @@ Sample client code:
text={AutocutTSCode}
startMarker="// START Autocut"
endMarker="// END Autocut"
- language="ts"
+ language="tsv2"
/>
@@ -341,7 +341,7 @@ The sort function takes either an object, or an array of objects, that describe
text={PyCodeV3}
startMarker="# START Sorting Python"
endMarker="# END Sorting Python"
- language="py"
+ language="pyv3"
/>
@@ -350,7 +350,7 @@ The sort function takes either an object, or an array of objects, that describe
text={TSCode}
startMarker="// START Sorting"
endMarker="// END Sorting"
- language="ts"
+ language="tsv2"
/>
@@ -442,7 +442,7 @@ To sort by more than one property, pass an array of { `path`, `order` } objects
text={PyCodeV3}
startMarker="# START MultiplePropSorting Python"
endMarker="# END MultiplePropSorting Python"
- language="py"
+ language="pyv3"
/>
@@ -451,7 +451,7 @@ To sort by more than one property, pass an array of { `path`, `order` } objects
text={TSCode}
startMarker="// START MultiplePropSorting"
endMarker="// END MultiplePropSorting"
- language="ts"
+ language="tsv2"
/>
@@ -518,7 +518,7 @@ To sort with metadata, add an underscore to the property name.
text={PyCodeV3}
startMarker="# START AdditionalPropSorting Python"
endMarker="# END AdditionalPropSorting Python"
- language="py"
+ language="pyv3"
/>
@@ -527,7 +527,7 @@ To sort with metadata, add an underscore to the property name.
text={TSCode}
startMarker="// START AdditionalPropSorting"
endMarker="// END AdditionalPropSorting"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/api/graphql/additional-properties.md b/developers/weaviate/api/graphql/additional-properties.md
index b969fb7ee0..9e079e91dc 100644
--- a/developers/weaviate/api/graphql/additional-properties.md
+++ b/developers/weaviate/api/graphql/additional-properties.md
@@ -1,5 +1,6 @@
---
title: Additional properties (metadata)
+description: Use additional properties in Weaviate's GraphQL API for enriched query results.
sidebar_position: 45
image: og/docs/api.jpg
# tags: ['graphql', 'additional properties', 'additional', 'metadata']
@@ -10,8 +11,6 @@ import TryEduDemo from '/_includes/try-on-edu-demo.mdx';
-## Overview
-
Various 'additional properties', also called 'metadata', can be retrieved in queries.
### Available additional properties
diff --git a/developers/weaviate/api/graphql/aggregate.md b/developers/weaviate/api/graphql/aggregate.md
index ae007cd3ed..f0050179a2 100644
--- a/developers/weaviate/api/graphql/aggregate.md
+++ b/developers/weaviate/api/graphql/aggregate.md
@@ -146,7 +146,7 @@ As such, this `Aggregate` query will retrieve the total object count in a class.
### groupBy argument
-You can use a groupBy argument to get meta information about groups of data objects.
+You can use a groupBy argument to get meta information about groups of data objects, from those matching a query. The groups can be based on a property of the data objects.
import GroupbyLimitations from '/_includes/groupby-limitations.mdx';
diff --git a/developers/weaviate/api/graphql/explore.md b/developers/weaviate/api/graphql/explore.md
index 37477e2b73..592e039e7c 100644
--- a/developers/weaviate/api/graphql/explore.md
+++ b/developers/weaviate/api/graphql/explore.md
@@ -14,8 +14,6 @@ As a result, `Explore` is not available on [Weaviate Cloud (WCD)](https://consol
:::
-## Overview
-
Use `Explore` to perform vector searches across multiple collections. Note that `Explore` is currently not available in the gRPC API.
### Requirements
diff --git a/developers/weaviate/api/graphql/filters.md b/developers/weaviate/api/graphql/filters.md
index 58039406fa..768767f4f6 100644
--- a/developers/weaviate/api/graphql/filters.md
+++ b/developers/weaviate/api/graphql/filters.md
@@ -1,5 +1,6 @@
---
title: Conditional filters
+description: Apply filters in Weaviate's GraphQL API to refine search and data queries.
sidebar_position: 35
image: og/docs/api.jpg
# tags: ['graphql', 'filters']
@@ -10,8 +11,6 @@ import TryEduDemo from '/_includes/try-on-edu-demo.mdx';
-## Overview
-
Conditional filters may be added to queries such as [`Object-level`](./get.md) and [`Aggregate`](./aggregate.md) queries, as well as [batch deletion](../../manage-data/delete.mdx#delete-multiple-objects). The operator used for filtering is also called a `where` filter.
A filter may consist of one or more conditions, which are combined using the `And` or `Or` operators. Each condition consists of a property path, an operator, and a value.
diff --git a/developers/weaviate/api/graphql/get.md b/developers/weaviate/api/graphql/get.md
index f3f8b1d057..412fcd8981 100644
--- a/developers/weaviate/api/graphql/get.md
+++ b/developers/weaviate/api/graphql/get.md
@@ -1,5 +1,6 @@
---
title: Object-level queries (Get)
+description: Fetch data from Weaviate with GraphQL's get queries for efficient retrieval.
sidebar_position: 10
image: og/docs/api.jpg
# tags: ['graphql', 'get{}']
@@ -11,8 +12,6 @@ import TryEduDemo from '/_includes/try-on-edu-demo.mdx';
-## Overview
-
This page covers object-level query functions. They are collectively referred to as `Get` queries within.
diff --git a/developers/weaviate/api/graphql/search-operators.md b/developers/weaviate/api/graphql/search-operators.md
index 3f6e754b40..ada5384d6e 100644
--- a/developers/weaviate/api/graphql/search-operators.md
+++ b/developers/weaviate/api/graphql/search-operators.md
@@ -1,5 +1,6 @@
---
title: Search operators
+description: Leverage search operators in Weaviate's GraphQL API for targeted data retrieval.
sidebar_position: 20
image: og/docs/api.jpg
# tags: ['graphql', 'search operators']
@@ -10,9 +11,6 @@ import TryEduDemo from '/_includes/try-on-edu-demo.mdx';
-
-## Overview
-
This page covers the search operators that can be used in queries, such as vector search operators (`nearText`, `nearVector`, `nearObject`, etc), keyword search operator (`bm25`), hybrid search operator (`hybrid`).
Only one search operator can be added to queries on the collection level.
diff --git a/developers/weaviate/api/grpc.md b/developers/weaviate/api/grpc.md
index 1ce5d2c3d6..e9d3d58c54 100644
--- a/developers/weaviate/api/grpc.md
+++ b/developers/weaviate/api/grpc.md
@@ -1,12 +1,11 @@
---
title: gRPC
+description: Integrate gRPC API with Weaviate for efficient data access.
sidebar_position: 5
image: og/docs/api.jpg
# tags: ['schema']
---
-## Overview
-
Starting with Weaviate `v1.19.0`, a gRPC interface has been progressively added to Weaviate. gRPC is a high-performance, open-source universal RPC framework that is contract-based and can be used in any environment. It is based on HTTP/2 and Protocol Buffers, and is therefore very fast and efficient.
As of Weaviate `v1.23.7`, the gRPC interface is considered stable. The latest (`v4`) Weaviate Python client library supports gRPC, and the other client libraries will follow.
diff --git a/developers/weaviate/api/index.md b/developers/weaviate/api/index.md
index d61715dae6..7e7b82c53d 100644
--- a/developers/weaviate/api/index.md
+++ b/developers/weaviate/api/index.md
@@ -1,5 +1,6 @@
---
title: Reference - API
+description: Weaviate API Documentation
sidebar_position: 0
image: og/docs/api.jpg
# tags: ['schema']
@@ -15,8 +16,6 @@ image: og/docs/api.jpg
- [Concepts: Interface](../concepts/interface.md)
:::
-## Overview
-
This section includes references for the RESTful and GraphQL APIs. RESTful API end-points can be used to add data or obtain information about the Weaviate instance, and the GraphQL interface to retrieve data.
### References - Search
diff --git a/developers/weaviate/benchmarks/ann.md b/developers/weaviate/benchmarks/ann.md
index f21865f310..5f3705ff90 100644
--- a/developers/weaviate/benchmarks/ann.md
+++ b/developers/weaviate/benchmarks/ann.md
@@ -8,15 +8,15 @@ image: og/docs/benchmarks.jpg
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
+import useBaseUrl from '@docusaurus/useBaseUrl';
+import ThemedImage from '@theme/ThemedImage';
+
+import BenchmarkGrid from '@site/src/components/Documentation/BenchmarkGrid';
+
# ANN Benchmark
This vector database benchmark is designed to measure and illustrate Weaviate's Approximate Nearest Neighbor (ANN) performance for a range of real-life use cases.
-:::note
-This is not a comparative benchmark that runs Weaviate against competing vector database solutions.
-To discuss trade-offs with other solutions, [contact sales](https://weaviate.io/pricing#contact-sales).
-:::
-
To make the most of this vector database benchmark, you can look at it from different perspectives:
- **The overall performance** – Review the [benchmark results](#benchmark-results) to draw conclusions about what to expect from Weaviate in a production setting.
@@ -48,11 +48,11 @@ Add a note about how many runs there were. Or, if there's only one run because o
For each set of parameters, we've run 10,000 requests, and we measured the following metrics:
-- The **Recall@1**, **Recall@10**, **Recall@100** - by comparing Weaviate's results to the ground truths specified in each dataset.
+- The **Recall@10** and **Recall@100** - by comparing Weaviate's results to the ground truths specified in each dataset.
- **Multi-threaded Queries per Second (QPS)** - The overall throughput you can
achieve with each configuration.
- **Individual Request Latency (mean)** - The mean latency over all 10,000 requests.
-- **P99 Latency** - 99% of all requests (9,900 out of 10,000) have a latency that is lower than or equal to this number – this shows how fast
+- **P99 Latency** - 99% of all requests (9,900 out of 10,000) have a latency that is lower than or equal to this number
- **Import time** - Since varying build parameters has an effect on import time, the import time is also included.
By request, we mean:
@@ -84,107 +84,148 @@ This section contains datasets modeled after the [ANN Benchmarks](https://github
| **Dataset** | **Number of Objects** | **Vector Dimensions** | **[Distance metric](https://weaviate.io/blog/distance-metrics-in-vector-search)** | **Use case** |
| --- | --- | --- | --- | --- |
-| [SIFT1M](http://corpus-texmex.irisa.fr/) | 1 M | 128 | Euclidean | This dataset reflects a common use case with a small number of objects. |
-| [Glove-25](https://nlp.stanford.edu/projects/glove/) | 1.28 M | 25 | Cosine | Because of the smaller vectors, Weaviate can achieve the highest throughput on this dataset. |
-| [Deep Image 96](https://sites.skoltech.ru/compvision/noimi/) | 10 M | 96 | Cosine | This dataset gives a good indication of expected speed and throughput when datasets grow. It is about 10 times larger than SIFT1M, but the throughput is only slightly lower. |
-| [GIST 960](http://corpus-texmex.irisa.fr/) | 1 M | 960 | Euclidean | This dataset highlights the cost of high-dimensional vector comparisons. It has the lowest throughput of the sample datasets. Use this one if you run high-dimensional loads. |
+| [SIFT1M](http://corpus-texmex.irisa.fr/) | 1 M | 128 | l2-squared | SIFT is a common ANN test dataset generated from image data |
+| [DBPedia OpenAI](https://huggingface.co/datasets/mteb/dbpedia) | 1 M | 1536 | cosine | DBPedia dataset from MTEB embedded with OpenAI's ada002 model |
+| [MSMARCO Snowflake](https://huggingface.co/datasets/Snowflake/mteb-retrieval-snowflake-arctic-embed-m-v1.5) | 8.8 M | 768 | l2-squared | MSMARCO dataset embedded with Snowflake's Arctic Embed M v1.5 model |
+| [Sphere DPR](https://weaviate.io/blog/sphere-dataset-in-weaviate) | 10 M | 768 | dot | Meta's Sphere dataset (10M sample) |
#### Benchmark Datasets
These are the results for each dataset:
-
-
-#### QPS vs Recall for SIFT1M
-
-
-
-import AnnSift128 from '/_includes/ann-sift-128.mdx';
-
-
-
-import AnnReadResultsTable from '/_includes/ann-read-results-table.mdx';
+
+
+#### QPS vs Recall for DBPedia OpenAI
+
+
+
+
+
+
+
+
+
+
+
-#### Recommended configuration for SIFT1M
-import RecommendedConfig from '/_includes/ann-recommended-config.mdx';
-
+#### Recommended configuration for DBPedia OpenAI ada002
| `efConstruction` | `maxConnections` | `ef` | **Recall@10** | **QPS (Limit 10)** | **Mean Latency (Limit 10**) | **p99 Latency (Limit 10)** |
| ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 128 | 32 | 64 | 98.83% | 8905 | 3.31ms | 4.49ms |
-
+| 256 | 16 | 96 | 97.24% | 5639 | 2.80ms | 4.43ms |
-
-
-#### QPS vs Recall for Glove-25
+
-
+#### QPS vs Recall for SIFT1M
-import AnnGlove25 from '/_includes/ann-glove-25.mdx';
+
+
+
+
+
+
+
+
+
+
-
+import AnnReadResultsTable from '/_includes/ann-read-results-table.mdx';
-#### Recommended configuration for Glove-25
+#### Recommended configuration for SIFT1M
+import RecommendedConfig from '/_includes/ann-recommended-config.mdx';
+
| `efConstruction` | `maxConnections` | `ef` | **Recall@10** | **QPS (Limit 10)** | **Mean Latency (Limit 10**) | **p99 Latency (Limit 10)** |
| ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 64 | 16 | 64 | 95.56% | 15003 | 1.93ms | 2.94ms |
+| 256 | 32 | 64 | 98.35% | 10940 | 1.44ms | 3.13ms |
-
-
-#### QPS vs Recall for Deep Image 96
-
-
-
-import AnnDeep96 from '/_includes/ann-deep-96.mdx';
-
-
+
+
+
+#### QPS vs Recall for MSMARCO Snowflake
+
+
+
+
+
+
+
+
+
+
+
-#### Recommended configuration for Deep Image 96
+#### Recommended configuration for MSMARCO Snowflake
| `efConstruction` | `maxConnections` | `ef` | **Recall@10** | **QPS (Limit 10)** | **Mean Latency (Limit 10**) | **p99 Latency (Limit 10)** |
| ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 128 | 32 | 64 | 96.43% | 6112 | 4.7ms | 15.87ms |
+| 384 | 32 | 48 | 97.36% | 7363 | 2.15ms | 3.69ms |
-
-
-#### QPS vs Recall for GIST 960
-
-
-
-import AnnGist960 from '/_includes/ann-gist-960.mdx';
-
-
+
+
+#### QPS vs Recall for Sphere DPR
+
+
+
+
+
+
+
+
+
+
+
-#### Recommended configuration for GIST 960
+#### Recommended configuration for Sphere DPR
| `efConstruction` | `maxConnections` | `ef` | **Recall@10** | **QPS (Limit 10)** | **Mean Latency (Limit 10**) | **p99 Latency (Limit 10)** |
| ----- | ----- | ----- | ----- | ----- | ----- | ----- |
-| 512 | 32 | 128 | 94.14% | 1935 | 15.05ms | 19.86ms |
+| 384 | 32 | 96 | 96.06% | 3523 | 4.49ms | 7.73ms |
-
-
-
## Benchmark Setup
### Scripts
@@ -195,12 +236,11 @@ This benchmark is [open source](https://github.com/weaviate/weaviate-benchmarkin
-This benchmark test uses two GCP instances within the same VPC:
+This benchmark test uses one GCP instances to run both Weaviate and the Benchmark scripts:
-* **Benchmark** – a `c2-standard-30` instance with 30 vCPU cores and 120 GB memory – to host Weaviate.
-* **Script** – a smaller instance with 8 vCPU – to run benchmarking scripts.
+* a `n4-highmem-16` instance with 16 vCPU cores and 128 GB memory.
-:::info Here's why we chose the `c2-standard-30`:
+:::info Here's why we chose the `n4-highmem-16`:
* It is large enough to show that Weaviate is a highly concurrent [vector search engine](https://weaviate.io/blog/what-is-a-vector-database).
* It scales well while running thousands of searches across multiple threads.
@@ -222,10 +262,10 @@ We modeled our dataset selection after
queries are used to test speed, throughput, and recall. The provided ground
truths are used to calculate the recall.
-We use Weaviate's Python client to import data.
+We use Weaviate's Golang client to import data.
We use Go to measure the concurrent (multi-threaded) queries.
- Each language has its own performance characteristics.
- You may get different results if you use a different language to send your queries.
+Each language has its own performance characteristics.
+You may get different results if you use a different language to send your queries.
For maximum throughput, we recommend using the [Go](/developers/weaviate/client-libraries/go.md) or
[Java](/developers/weaviate/client-libraries/java.md) client libraries.
diff --git a/developers/weaviate/benchmarks/img/benchmark_deepimage_96.png b/developers/weaviate/benchmarks/img/benchmark_deepimage_96.png
deleted file mode 100644
index 8cc772caea..0000000000
Binary files a/developers/weaviate/benchmarks/img/benchmark_deepimage_96.png and /dev/null differ
diff --git a/developers/weaviate/benchmarks/img/benchmark_gist_960.png b/developers/weaviate/benchmarks/img/benchmark_gist_960.png
deleted file mode 100644
index 2236d6f416..0000000000
Binary files a/developers/weaviate/benchmarks/img/benchmark_gist_960.png and /dev/null differ
diff --git a/developers/weaviate/benchmarks/img/benchmark_glove_25.png b/developers/weaviate/benchmarks/img/benchmark_glove_25.png
deleted file mode 100644
index 285561f3a7..0000000000
Binary files a/developers/weaviate/benchmarks/img/benchmark_glove_25.png and /dev/null differ
diff --git a/developers/weaviate/benchmarks/img/benchmark_sift_128.png b/developers/weaviate/benchmarks/img/benchmark_sift_128.png
deleted file mode 100644
index 2c88c52b6c..0000000000
Binary files a/developers/weaviate/benchmarks/img/benchmark_sift_128.png and /dev/null differ
diff --git a/developers/weaviate/client-libraries/go.md b/developers/weaviate/client-libraries/go.md
index f7f67609b3..906ab0b586 100644
--- a/developers/weaviate/client-libraries/go.md
+++ b/developers/weaviate/client-libraries/go.md
@@ -411,73 +411,16 @@ client.Data().Validator().WithProperties(properties)
## Releases
-For links to the Go Client releases, expand this section.
+Go to the [GitHub releases page](https://github.com/weaviate/weaviate-go-client/releases) to see the history of the Go client library releases.
- Go Client
-
-| Client Version | Release Date |
-| :- | :- |
-| [4.15.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.15.1) | 2024-08-07 |
-| [4.15.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.15.0) | 2024-07-25 |
-| [4.14.3](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.14.3) | 2024-07-25 |
-| [4.14.2](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.14.2) | 2024-07-17 |
-| [4.14.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.14.1) | 2024-07-11 |
-| [4.14.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.14.0) | 2024-05-20 |
-| [4.13.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.13.1) | 2024-03-07 |
-| [4.13.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.13.0) | 2024-03-05 |
-| [4.12.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.12.0) | 2023-12-08 |
-| [4.11.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.11.0) | 2023-11-13 |
-| [4.10.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.10.0) | 2023-08-22 |
-| [4.9.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.9.0) | 2023-07-06 |
-| [4.8.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.8.1) | 2023-05-26 |
-| [4.8.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.8.0) | 2023-05-05 |
-| [4.7.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.7.1) | 2023-04-14 |
-| [4.7.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.7.0) | 2023-04-03 |
-| [4.6.4](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.6.4) | 2023-03-25 |
-| [4.6.3](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.6.3) | 2023-03-14 |
-| [4.6.2](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.6.2) | 2023-03-07 |
-| [4.6.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.6.1) | 2023-02-01 |
-| [4.6.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.6.0) | 2023-02-01 |
-| [4.5.2](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.5.2) | 2023-01-19 |
-| [4.5.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.5.1) | 2023-01-17 |
-| [4.5.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.5.0) | 2022-12-20 |
-| [4.4.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.4.0) | 2022-10-31 |
-| [4.3.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.3.1) | 2022-09-27 |
-| [4.3.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.3.0) | 2022-09-07 |
-| [4.2.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.2.1) | 2022-07-12 |
-| [4.2.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.2.0) | 2022-07-07 |
-| [4.1.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.1.0) | 2022-05-25 |
-| [4.0.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v4.0.0) | 2022-05-03 |
-| [3.0.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v3.0.0) | 2022-04-05 |
-| [2.6.2](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.6.2) | 2022-10-31 |
-| [2.6.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.6.1) | 2022-05-04 |
-| [2.6.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.6.0) | 2022-03-11 |
-| [2.5.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.5.0) | 2022-02-01 |
-| [2.4.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.4.1) | 2021-12-15 |
-| [2.4.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.4.0) | 2021-11-30 |
-| [2.3.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.3.0) | 2021-08-31 |
-| [2.2.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.2.0) | 2021-06-07 |
-| [2.1.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.1.0) | 2021-04-23 |
-| [2.0.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.0.1) | 2021-03-25 |
-| [2.0.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v2.0.0) | 2021-03-23 |
-| [1.1.2](https://github.com/weaviate/weaviate-go-client/releases/tag/v1.1.2) | 2021-10-31 |
-| [1.1.1](https://github.com/weaviate/weaviate-go-client/releases/tag/v1.1.1) | 2021-05-04 |
-| [1.1.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v1.1.0) | 2020-11-09 |
-| [1.0.0](https://github.com/weaviate/weaviate-go-client/releases/tag/v1.0.0) | 2020-11-06 |
+ Click here for a table of Weaviate and corresponding client versions
-
-
-import MatrixIntro from '/_includes/clients/matrix-intro.md';
-
-
+import ReleaseHistory from '/_includes/release-history.md';
-## Change logs
-
-Check the
-[change logs on GitHub](https://github.com/weaviate/weaviate-go-client/releases)
-for updates on the latest `Go client` changes.
+
+
## Questions and feedback
diff --git a/developers/weaviate/client-libraries/index.md b/developers/weaviate/client-libraries/index.md
index 4ba1ddc9e4..e2d2551a6c 100644
--- a/developers/weaviate/client-libraries/index.md
+++ b/developers/weaviate/client-libraries/index.md
@@ -1,13 +1,11 @@
---
title: Reference - Client Libraries
+description: Explore Weaviate client libraries to integrate Weaviate into your tech stack.
sidebar_position: 0
image: og/docs/client-libraries.jpg
# tags: ['client libraries', 'cli']
---
-
-## Overview
-
You can interact with Weaviate by using the GraphQL or RESTful API directly, or with one of the available client libraries.
Currently, Weaviate supports:
diff --git a/developers/weaviate/client-libraries/java.md b/developers/weaviate/client-libraries/java.md
index dfea34dc4b..2896fb3796 100644
--- a/developers/weaviate/client-libraries/java.md
+++ b/developers/weaviate/client-libraries/java.md
@@ -423,80 +423,16 @@ client.classifications().scheduler().withTrainingSetWhereFilter(where)...
## Releases
-For links to the Go Client releases, expand this section.
+Go to the [GitHub releases page](https://github.com/weaviate/java-client/releases) to see the history of the Java client library releases.
- Go Client
-
-| Client Version | Release Date |
-| :- | :- |
-| [4.8.3](https://github.com/weaviate/java-client/releases/tag/4.8.3) | 2024-09-21 |
-| [4.8.2](https://github.com/weaviate/java-client/releases/tag/4.8.2) | 2024-07-24 |
-| [4.8.1](https://github.com/weaviate/java-client/releases/tag/4.8.1) | 2024-07-24 |
-| [4.8.0](https://github.com/weaviate/java-client/releases/tag/4.8.0) | 2024-07-23 |
-| [4.7.0](https://github.com/weaviate/java-client/releases/tag/4.7.0) | 2024-05-20 |
-| [4.6.0](https://github.com/weaviate/java-client/releases/tag/4.6.0) | 2024-03-05 |
-| [4.5.1](https://github.com/weaviate/java-client/releases/tag/4.5.1) | 2024-01-08 |
-| [4.5.0](https://github.com/weaviate/java-client/releases/tag/4.5.0) | 2023-12-07 |
-| [4.4.2](https://github.com/weaviate/java-client/releases/tag/4.4.2) | 2023-11-20 |
-| [4.4.1](https://github.com/weaviate/java-client/releases/tag/4.4.1) | 2023-11-15 |
-| [4.4.0](https://github.com/weaviate/java-client/releases/tag/4.4.0) | 2023-11-14 |
-| [4.3.0](https://github.com/weaviate/java-client/releases/tag/4.3.0) | 2023-08-22 |
-| [4.2.1](https://github.com/weaviate/java-client/releases/tag/4.2.1) | 2023-07-26 |
-| [4.2.0](https://github.com/weaviate/java-client/releases/tag/4.2.0) | 2023-07-06 |
-| [4.1.2](https://github.com/weaviate/java-client/releases/tag/4.1.2) | 2023-05-26 |
-| [4.1.1](https://github.com/weaviate/java-client/releases/tag/4.1.1) | 2023-05-21 |
-| [4.1.0](https://github.com/weaviate/java-client/releases/tag/4.1.0) | 2023-05-05 |
-| [4.0.2](https://github.com/weaviate/java-client/releases/tag/4.0.2) | 2023-03-28 |
-| [4.0.1](https://github.com/weaviate/java-client/releases/tag/4.0.1) | 2023-03-16 |
-| [4.0.0](https://github.com/weaviate/java-client/releases/tag/4.0.0) | 2023-03-14 |
-| [3.6.5](https://github.com/weaviate/java-client/releases/tag/3.6.5) | 2023-03-13 |
-| [3.6.4](https://github.com/weaviate/java-client/releases/tag/3.6.4) | 2023-03-07 |
-| [3.6.3](https://github.com/weaviate/java-client/releases/tag/3.6.3) | 2023-03-01 |
-| [3.6.2](https://github.com/weaviate/java-client/releases/tag/3.6.2) | 2023-02-23 |
-| [3.6.1](https://github.com/weaviate/java-client/releases/tag/3.6.1) | 2023-02-22 |
-| [3.6.0](https://github.com/weaviate/java-client/releases/tag/3.6.0) | 2023-02-09 |
-| [3.5.0](https://github.com/weaviate/java-client/releases/tag/3.5.0) | 2022-12-20 |
-| [3.4.2](https://github.com/weaviate/java-client/releases/tag/3.4.2) | 2022-11-21 |
-| [3.4.1](https://github.com/weaviate/java-client/releases/tag/3.4.1) | 2022-11-18 |
-| [3.4.0](https://github.com/weaviate/java-client/releases/tag/3.4.0) | 2022-10-31 |
-| [3.3.2](https://github.com/weaviate/java-client/releases/tag/3.3.2) | 2022-10-29 |
-| [3.3.1](https://github.com/weaviate/java-client/releases/tag/3.3.1) | 2022-09-29 |
-| [3.3.0](https://github.com/weaviate/java-client/releases/tag/3.3.0) | 2022-09-07 |
-| [3.2.1](https://github.com/weaviate/java-client/releases/tag/3.2.1) | 2022-07-12 |
-| [3.2.0](https://github.com/weaviate/java-client/releases/tag/3.2.0) | 2022-07-07 |
-| [3.1.1](https://github.com/weaviate/java-client/releases/tag/3.1.1) | 2022-06-02 |
-| [3.1.0](https://github.com/weaviate/java-client/releases/tag/3.1.0) | 2022-05-25 |
-| [3.0.0](https://github.com/weaviate/java-client/releases/tag/3.0.0) | 2022-05-04 |
-| [2.4.0](https://github.com/weaviate/java-client/releases/tag/2.4.0) | 2022-04-05 |
-| [2.3.0](https://github.com/weaviate/java-client/releases/tag/2.3.0) | 2022-03-11 |
-| [2.2.2](https://github.com/weaviate/java-client/releases/tag/2.2.2) | 2022-02-08 |
-| [2.2.1](https://github.com/weaviate/java-client/releases/tag/2.2.1) | 2022-02-07 |
-| [2.2.0](https://github.com/weaviate/java-client/releases/tag/2.2.0) | 2022-02-01 |
-| [2.1.1](https://github.com/weaviate/java-client/releases/tag/2.1.1) | 2021-12-15 |
-| [2.1.0](https://github.com/weaviate/java-client/releases/tag/2.1.0) | 2021-11-30 |
-| [2.0.0](https://github.com/weaviate/java-client/releases/tag/2.0.0) | 2021-11-04 |
-| [1.2.1](https://github.com/weaviate/java-client/releases/tag/1.2.1) | 2021-09-28 |
-| [1.2.0](https://github.com/weaviate/java-client/releases/tag/1.2.0) | 2021-09-10 |
-| [1.1.2](https://github.com/weaviate/java-client/releases/tag/1.1.2) | 2021-08-31 |
-| [1.1.1](https://github.com/weaviate/java-client/releases/tag/1.1.1) | 2021-08-10 |
-| [1.1.0](https://github.com/weaviate/java-client/releases/tag/1.1.0) | 2021-06-07 |
-| [1.0.2](https://github.com/weaviate/java-client/releases/tag/1.0.2) | 2021-04-26 |
-| [1.0.1](https://github.com/weaviate/java-client/releases/tag/1.0.1) | 2021-04-23 |
-| [1.0.0](https://github.com/weaviate/java-client/releases/tag/1.0.0) | 2021-04-22 |
+ Click here for a table of Weaviate and corresponding client versions
-
-
-import MatrixIntro from '/_includes/clients/matrix-intro.md';
-
-
+import ReleaseHistory from '/_includes/release-history.md';
-## Change logs
-
-Check the
-[change logs on GitHub](https://github.com/weaviate/weaviate-java-client/releases)
-for updates on the latest `Java client` changes.
+
+
## Questions and feedback
diff --git a/developers/weaviate/client-libraries/python/index.md b/developers/weaviate/client-libraries/python/index.md
index 7ee3cfc2cc..fb73865bea 100644
--- a/developers/weaviate/client-libraries/python/index.md
+++ b/developers/weaviate/client-libraries/python/index.md
@@ -1,5 +1,6 @@
---
title: Python
+description: Utilize the Python client library to access Weaviate and streamline data processes.
sidebar_position: 10
image: og/docs/client-libraries.jpg
# tags: ['python', 'client library', 'experimental']
@@ -11,10 +12,12 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PythonCode from '!!raw-loader!/_includes/code/client-libraries/python_v4.py';
import BatchVectorCode from '!!raw-loader!/_includes/code/howto/manage-data.import.py';
-## Overview
-
This page broadly covers the Weaviate Python client (`v4` release). For usage information not specific to the Python client, such as code examples, see the relevant pages in the [Weaviate documentation](../../index.md). Some frequently used sections are [listed here](#code-examples--resources) for convenience.
+:::info Inline documentation
+For the inline documentation (also known as docstrings) of the Python client, see the [Read the Docs](https://weaviate-python-client.readthedocs.io/en/latest/) page.
+:::
+
### Asynchronous usage
The Python client library provides a synchronous API by default through the `WeaviateClient` class, which is covered on this page. An asynchronous API is also available through the `WeaviateAsyncClient` class (from `weaviate-client` `v4.7.0` and up). See the [async client API page](./async.md) for further details.
@@ -28,7 +31,8 @@ If you are migrating from the `v3` client to the `v4`, see this [dedicated guide
The Python client library is developed and tested using Python 3.8+. It is available on [PyPI.org](https://pypi.org/project/weaviate-client/), and can be installed with:
```bash
-pip install -U weaviate-client # For beta versions: `pip install --pre -U "weaviate-client==4.*"`
+# For beta versions: `pip install --pre -U "weaviate-client==4.*"`
+pip install -U weaviate-client
```
### Requirements
@@ -463,8 +467,9 @@ The `data` submodule contains all object-level CUD operations, including:
- `insert` for creating objects.
- This function takes the object properties as a dictionary.
-- `insert_many` for batch creating multiple objects.
+- `insert_many` for adding multiple objects with one request.
- This function takes the object properties as a dictionary or as a `DataObject` instance.
+ - Note: For larger numbers of objects, consider using [batch imports](#batch-imports).
- `update` for updating objects (for `PATCH` operations).
- `replace` for replacing objects (for `PUT` operations).
- `delete_by_id` for deleting objects by ID.
@@ -473,11 +478,6 @@ The `data` submodule contains all object-level CUD operations, including:
See some examples below. Note that each function will return varying types of objects.
-:::caution `insert_many` sends one request
-As of `4.4b1`, `insert_many` sends one request for the entire function call. A future release may
-send multiple requests as batches.
-:::
-
@@ -521,6 +521,10 @@ send multiple requests as batches.
+:::info `insert_many` sends one request
+`insert_many` sends one request for the entire function call. For requests with a large number of objects, [consider using `batch` imports](#batch-imports).
+:::
+
#### `insert_many` with DataObjects
The `insert_many` function takes a list of `DataObject` instances or a list of dictionaries. This is useful if you want to specify additional information to the properties, such as cross-references, object uuid, or a custom vector.
@@ -1202,121 +1206,16 @@ The Weaviate API reference pages for [search](../../api/graphql/index.md) and [R
## Releases
-For links to the Python Client releases, expand this section.
+Go to the [GitHub releases page](https://github.com/weaviate/weaviate-python-client/releases) to see the history of the Python client library releases.
- Python Client
-
-| Client Version | Release Date |
-| :- | :- |
-| [4.8.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.8.1) | 2024-09-16 |
-| [4.8.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.8.0) | 2024-09-10 |
-| [4.7.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.7.1) | 2024-07-25 |
-| [4.7.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.7.0) | 2024-07-24 |
-| [4.6.7](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.7) | 2024-07-19 |
-| [4.6.6](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.6) | 2024-07-02 |
-| [4.6.5](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.5) | 2024-06-19 |
-| [4.6.4](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.4) | 2024-06-19 |
-| [4.6.3](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.3) | 2024-05-21 |
-| [4.6.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.2) | 2024-05-21 |
-| [4.6.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.1) | 2024-05-17 |
-| [4.6.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.6.0) | 2024-05-10 |
-| [4.5.7](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.7) | 2024-05-03 |
-| [4.5.6](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.6) | 2024-04-23 |
-| [4.5.5](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.5) | 2024-04-03 |
-| [4.5.4](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.4) | 2024-03-15 |
-| [4.5.3](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.3) | 2024-03-14 |
-| [4.5.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.2) | 2024-03-11 |
-| [4.5.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.1) | 2024-03-04 |
-| [4.5.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.5.0) | 2024-02-27 |
-| [4.4.4](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.4.4) | 2024-02-09 |
-| [4.4.3](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.4.3) | 2024-02-09 |
-| [4.4.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.4.2) | 2024-02-09 |
-| [4.4.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.4.1) | 2024-02-05 |
-| [4.4.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.4.0) | 2024-01-31 |
-| [3.26.7](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.6) | 2024-08-15 |
-| [3.26.6](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.6) | 2024-07-24 |
-| [3.26.5](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.5) | 2024-07-02 |
-| [3.26.4](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.4) | 2024-07-02 |
-| [3.26.3](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.3) | 2024-06-30 |
-| [3.26.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.2) | 2024-01-22 |
-| [3.26.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.1) | 2024-01-15 |
-| [3.26.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.26.0) | 2023-12-20 |
-| [3.25.3](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.25.3) | 2023-11-07 |
-| [3.25.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.25.2) | 2023-10-31 |
-| [3.25.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.25.1) | 2023-10-27 |
-| [3.25.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.25.0) | 2023-10-27 |
-| [3.24.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.24.2) | 2023-10-04 |
-| [3.24.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.24.1) | 2023-09-11 |
-| [3.24.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.24.0) | 2023-09-11 |
-| [3.23.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.23.2) | 2023-08-29 |
-| [3.23.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.23.1) | 2023-08-25 |
-| [3.23.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.23.0) | 2023-08-22 |
-| [3.22.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.22.1) | 2023-07-10 |
-| [3.22.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.22.0) | 2023-07-06 |
-| [3.21.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.21.0) | 2023-06-18 |
-| [3.20.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.20.1) | 2023-06-14 |
-| [3.20.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.20.0) | 2023-06-12 |
-| [3.19.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.19.2) | 2023-05-25 |
-| [3.19.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.19.1) | 2023-05-18 |
-| [3.19.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.19.0) | 2023-05-18 |
-| [3.18.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.19.0) | 2023-05-09 |
-| [3.17.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.17.1) | 2023-05-08 |
-| [3.17.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.17.0) | 2023-05-04 |
-| [3.16.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.16.2) | 2023-04-26 |
-| [3.16.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.16.1) | 2023-04-24 |
-| [3.16.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.16.0) | 2023-04-24 |
-| [3.15.6](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.15.6) | 2023-04-15 |
-| [3.15.5](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.15.5) | 2023-04-09 |
-| [3.15.4](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.15.4) | 2023-04-08 |
-| [3.15.3](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.15.3) | 2023-03-23 |
-| [3.15.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.15.2) | 2023-03-15 |
-| [3.15.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.15.1) | 2023-03-13 |
-| [3.15.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.15.0) | 2023-03-12 |
-| [3.14.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.14.0) | 2023-03-07 |
-| [3.13.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.13.0) | 2023-03-02 |
-| [3.12.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.12.0) | 2023-02-24 |
-| [3.11.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.11.0) | 2023-01-20 |
-| [3.10.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.10.0) | 2022-12-21 |
-| [3.9.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.9.0) | 2022-11-09 |
-| [3.8.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.8.0) | 2022-09-07 |
-| [3.7.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.7.0) | 2022-07-29 |
-| [3.6.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.6.0) | 2022-07-06 |
-| [3.5.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.5.1) | 2022-05-18 |
-| [3.5.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.5.0) | 2022-05-08 |
-| [3.4.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.4.2) | 2022-04-12 |
-| [3.4.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.4.1) | 2022-04-06 |
-| [3.4.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.4.0) | 2022-04-04 |
-| [3.2.5](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.2.5) | 2021-10-26 |
-| [3.2.4](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.2.4) | 2021-10-26 |
-| [3.2.3](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.2.3) | 2021-10-13 |
-| [3.2.2](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.2.2) | 2021-09-27 |
-| [3.2.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.2.1) | 2021-09-02 |
-| [3.2.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.2.0) | 2021-09-02 |
-| [3.1.1](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.1.1) | 2021-08-24 |
-| [3.1.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.1.0) | 2021-08-17 |
-| [3.0.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v3.0.0) | 2021-08-17 |
-| [2.5.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v2.5.0) | 2021-06-03 |
-| [2.4.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v2.4.0) | 2021-04-23 |
-| [2.3.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v2.3.0) | 2021-03-26 |
-| [2.2.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v2.2.0) | 2021-02-17 |
-| [2.1.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v2.1.0) | 2021-02-08 |
-| [2.0.0](https://github.com/weaviate/weaviate-python-client/releases/tag/v2.0.0) | 2021-01-11 |
+ Click here for a table of Weaviate and corresponding client versions
-
-
-import MatrixIntro from '/_includes/clients/matrix-intro.md';
-
-
+import ReleaseHistory from '/_includes/release-history.md';
-## Change logs
-
-For more detailed information on client updates, check the change logs. The logs
-are hosted here:
-
-- [GitHub](https://github.com/weaviate/weaviate-python-client/releases)
-- [Read the Docs](https://weaviate-python-client.readthedocs.io/en/stable/changelog.html)
+
+
## Questions and feedback
diff --git a/developers/weaviate/client-libraries/python/python_v3.md b/developers/weaviate/client-libraries/python/python_v3.md
index 4e4ada53f0..8e50610395 100644
--- a/developers/weaviate/client-libraries/python/python_v3.md
+++ b/developers/weaviate/client-libraries/python/python_v3.md
@@ -5,11 +5,11 @@ image: og/docs/client-libraries.jpg
# tags: ['python', 'client library']
---
-:::info `v3` client is deprecated
+:::caution `v3` client is deprecated
This document relates to the legacy `v3` client and API.
-Starting in December 2024, the `v4` [client installations](https://pypi.org/project/weaviate-client/) will no longer include the `v3` API (i.e. the `weaviate.Client` class). This will help us to provide the best developer experience for you, support for the latest features, and clearly separate the two.
+From `v4.10.0`, the [Weaviate Python client installations](https://pypi.org/project/weaviate-client/) no longer include the `v3` API (i.e. the `weaviate.Client` class). This separation helps us to provide the best developer experience for you and provide support for the latest Weaviate features.
The `v3` client will continue to get critical security updates and bugfixes for the foreseeable future, but it will not support any new features.
@@ -899,20 +899,18 @@ The batching algorithm in our client is not thread-safe. Keep this in mind to he
If you are performing batching in a multi-threaded scenario, ensure that only one of the threads is performing the batching workflow at any given time. No two threads can use the same `client.batch` object at one time.
-## Client releases
+## Releases
-import MatrixIntro from '/_includes/clients/matrix-intro.md';
+Go to the [GitHub releases page](https://github.com/weaviate/weaviate-python-client/releases) to see the history of the Python client library releases.
-
-
-## Change logs
+
+ Click here for a table of Weaviate and corresponding client versions
-For more detailed information on client updates, check the change logs. The logs
-are hosted here:
+import ReleaseHistory from '/_includes/release-history.md';
-- [GitHub](https://github.com/weaviate/weaviate-python-client/releases)
-- [Read the Docs](https://weaviate-python-client.readthedocs.io/en/stable/changelog.html)
+
+
## Questions and feedback
diff --git a/developers/weaviate/client-libraries/python/v3_v4_migration.md b/developers/weaviate/client-libraries/python/v3_v4_migration.md
index ecd1ad7231..263a97da01 100644
--- a/developers/weaviate/client-libraries/python/v3_v4_migration.md
+++ b/developers/weaviate/client-libraries/python/v3_v4_migration.md
@@ -32,8 +32,9 @@ To go from `v3` to `v4`, you must
pip install -U weaviate-client
```
-2. Upgrade Weaviate to a compatible version
- - Weaviate `1.23.7` is required for `v4.4.1`. Generally, we recommend you use the latest versions of Weaviate and the client.
+2. Upgrade Weaviate to a compatible version. Each minor Python client version is closely tied to a minor Weaviate version.
+ - For example, Weaviate `v1.27.x` were developed together with `v4.9.x` of the Python client.
+ - Generally, we recommend you use the latest versions of Weaviate and the client. You can find the version compatibility matrix in the [release notes](../../release-notes/index.md#weaviate-core-and-client-releases)
3. Make sure a port for gRPC is open to Weaviate.
- The default port is 50051.
@@ -193,7 +194,7 @@ import ManageDataCodeV3 from '!!raw-loader!/_includes/code/howto/manage-data.rea
text={ManageDataCodeV3}
startMarker="# ReadObject START"
endMarker="# ReadObject END"
- language="py"
+ language="pyv3"
/>
@@ -246,7 +247,7 @@ import SearchSimilarityCodeV3 from '!!raw-loader!/_includes/code/howto/search.si
text={SearchSimilarityCodeV3}
startMarker="# GetNearTextPython"
endMarker="# END GetNearTextPython"
- language="python"
+ language="pyv3"
/>
diff --git a/developers/weaviate/client-libraries/typescript/index.mdx b/developers/weaviate/client-libraries/typescript/index.mdx
index e677de0016..6cca085559 100644
--- a/developers/weaviate/client-libraries/typescript/index.mdx
+++ b/developers/weaviate/client-libraries/typescript/index.mdx
@@ -31,48 +31,16 @@ The v3 TypeScript client replaces the [v2 JavaScript](https://www.npmjs.com/pack
## Releases
-For links to the TypeScript Client releases, expand this section.
+Go to the [GitHub releases page](https://github.com/weaviate/typescript-client/releases) to see the history of the TypeScript client library releases.
- TypeScript Client
-
-| Client Version | Release Date |
-| :- | :- |
-| [3.1.0](https://github.com/weaviate/typescript-client/releases/tag/v3.1.0) | 2024-07-24 |
-| [3.0.9](https://github.com/weaviate/typescript-client/releases/tag/v3.0.9) | 2024-07-04 |
-| [3.0.8](https://github.com/weaviate/typescript-client/releases/tag/v3.0.8) | 2024-06-24 |
-| [3.0.7](https://github.com/weaviate/typescript-client/releases/tag/v3.0.7) | 2024-06-21 |
-| [3.0.6](https://github.com/weaviate/typescript-client/releases/tag/v3.0.6) | 2024-06-19 |
-| [3.0.5](https://github.com/weaviate/typescript-client/releases/tag/v3.0.5) | 2024-06-13 |
-| [3.0.4](https://github.com/weaviate/typescript-client/releases/tag/v3.0.4) | 2024-06-13 |
-| [3.0.3](https://github.com/weaviate/typescript-client/releases/tag/v3.0.3) | 2024-06-04 |
-| [3.0.2](https://github.com/weaviate/typescript-client/releases/tag/v3.0.2) | 2024-06-03 |
-| [3.0.1](https://github.com/weaviate/typescript-client/releases/tag/v3.0.1) | 2024-06-03 |
-| [3.0.0](https://github.com/weaviate/typescript-client/releases/tag/v3.0.0) | 2024-06-03 |
-| [2.2.0](https://github.com/weaviate/typescript-client/releases/tag/v2.2.0) | 2024-05-20 |
-| [2.1.1](https://github.com/weaviate/typescript-client/releases/tag/v2.1.1) | 2024-03-26 |
-| [2.1.0](https://github.com/weaviate/typescript-client/releases/tag/v2.1.0) | 2024-02-28 |
-| [2.0.0](https://github.com/weaviate/typescript-client/releases/tag/v2.0.0) | 2023-11-22 |
-| [1.6.0](https://github.com/weaviate/typescript-client/releases/tag/v1.6.0) | 2023-10-31 |
-| [1.5.0](https://github.com/weaviate/typescript-client/releases/tag/v1.5.0) | 2023-08-22 |
-| [1.4.0](https://github.com/weaviate/typescript-client/releases/tag/v1.4.0) | 2023-07-06 |
-| [1.3.3](https://github.com/weaviate/typescript-client/releases/tag/v1.3.3) | 2023-06-14 |
-| [1.3.2](https://github.com/weaviate/typescript-client/releases/tag/v1.3.2) | 2023-05-26 |
-| [1.3.1](https://github.com/weaviate/typescript-client/releases/tag/v1.3.1) | 2023-05-16 |
-| [1.3.0](https://github.com/weaviate/typescript-client/releases/tag/v1.3.0) | 2023-05-08 |
-| [1.2.0](https://github.com/weaviate/typescript-client/releases/tag/v1.2.0) | 2023-05-05 |
-| [1.1.0](https://github.com/weaviate/typescript-client/releases/tag/v1.1.0) | 2023-04-21 |
-| [1.0.0](https://github.com/weaviate/typescript-client/releases/tag/v1.0.0) | 2023-05-04 |
+ Click here for a table of Weaviate and corresponding client versions
-
-
-import MatrixIntro from '/_includes/clients/matrix-intro.md';
+import ReleaseHistory from '/_includes/release-history.md';
-
+
-## Client change logs
-
-See the client [change logs on GitHub](https://github.com/weaviate/typescript-client/releases).
+
## Questions and feedback
diff --git a/developers/weaviate/client-libraries/typescript/typescript-v2.mdx b/developers/weaviate/client-libraries/typescript/typescript-v2.mdx
index d222af7b48..dee9efa6c9 100644
--- a/developers/weaviate/client-libraries/typescript/typescript-v2.mdx
+++ b/developers/weaviate/client-libraries/typescript/typescript-v2.mdx
@@ -1,5 +1,6 @@
---
title: JS/TS client v2
+description: TypeScript Client Library v2
sidebar_position: 20
image: og/docs/client-libraries.jpg
# tags: ['TypeScript', 'client library']
@@ -419,9 +420,14 @@ The output looks like this.
## Client releases
-import MatrixIntro from '/_includes/clients/matrix-intro.md';
+
+ Click here for a table of Weaviate and corresponding client versions
+
+import ReleaseHistory from '/_includes/release-history.md';
-
+
+
+
## Client change logs
diff --git a/developers/weaviate/client-libraries/typescript/typescript-v3.mdx b/developers/weaviate/client-libraries/typescript/typescript-v3.mdx
index dd4a9436d2..672744ee92 100644
--- a/developers/weaviate/client-libraries/typescript/typescript-v3.mdx
+++ b/developers/weaviate/client-libraries/typescript/typescript-v3.mdx
@@ -1,5 +1,6 @@
---
title: JS/TS client v3
+description: TypeScript Client Library v3
sidebar_position: 10
image: og/docs/client-libraries.jpg
# tags: ['typescript', 'javascript', 'client library', 'experimental']
@@ -312,48 +313,16 @@ There are demo applications written in TypeScript and JavaScript here:
## Client releases
-For links to the TypeScript Client releases, expand this section.
+Go to the [GitHub releases page](https://github.com/weaviate/typescript-client/releases) to see the history of the TypeScript client library releases.
- TypeScript Client
-
-| Client Version | Release Date |
-| :- | :- |
- | [3.1.4](https://github.com/weaviate/typescript-client/releases/tag/v3.1.4) | 2024-08-06 |
- | [3.1.3](https://github.com/weaviate/typescript-client/releases/tag/v3.1.3) | 2024-07-26 |
- | [3.1.2](https://github.com/weaviate/typescript-client/releases/tag/v3.1.2) | 2024-07-26 |
- | [3.1.1](https://github.com/weaviate/typescript-client/releases/tag/v3.1.1) | 2024-07-26 |
- | [3.1.0](https://github.com/weaviate/typescript-client/releases/tag/v3.1.0) | 2024-07-24 |
- | [3.0.9](https://github.com/weaviate/typescript-client/releases/tag/v3.0.9) | 2024-07-10 |
- | [3.0.8](https://github.com/weaviate/typescript-client/releases/tag/v3.0.8) | 2024-06-24 |
- | [3.0.7](https://github.com/weaviate/typescript-client/releases/tag/v3.0.7) | 2024-06-21 |
- | [3.0.6](https://github.com/weaviate/typescript-client/releases/tag/v3.0.6) | 2024-06-19 |
- | [3.0.5](https://github.com/weaviate/typescript-client/releases/tag/v3.0.5) | 2024-06-13 |
- | [3.0.4](https://github.com/weaviate/typescript-client/releases/tag/v3.0.4) | 2024-06-13 |
- | [3.0.3](https://github.com/weaviate/typescript-client/releases/tag/v3.0.3) | 2024-06-04 |
- | [3.0.2](https://github.com/weaviate/typescript-client/releases/tag/v3.0.2) | 2024-06-03 |
- | [3.0.1](https://github.com/weaviate/typescript-client/releases/tag/v3.0.1) | 2024-06-03 |
- | [3.0.0](https://github.com/weaviate/typescript-client/releases/tag/v3.0.0) | 2024-06-03 |
- | [2.2.0](https://github.com/weaviate/typescript-client/releases/tag/v2.2.0) | 2024-05-20 |
- | [2.1.1](https://github.com/weaviate/typescript-client/releases/tag/v2.1.1) | 2024-03-26 |
- | [2.1.0](https://github.com/weaviate/typescript-client/releases/tag/v2.1.0) | 2024-02-28 |
- | [2.0.0](https://github.com/weaviate/typescript-client/releases/tag/v2.0.0) | 2023-11-22 |
- | [1.6.0](https://github.com/weaviate/typescript-client/releases/tag/v1.6.0) | 2023-10-31 |
- | [1.5.0](https://github.com/weaviate/typescript-client/releases/tag/v1.5.0) | 2023-08-22 |
- | [1.4.0](https://github.com/weaviate/typescript-client/releases/tag/v1.4.0) | 2023-07-06 |
- | [1.3.3](https://github.com/weaviate/typescript-client/releases/tag/v1.3.3) | 2023-06-14 |
- | [1.3.2](https://github.com/weaviate/typescript-client/releases/tag/v1.3.2) | 2023-05-26 |
- | [1.3.1](https://github.com/weaviate/typescript-client/releases/tag/v1.3.1) | 2023-05-16 |
- | [1.3.0](https://github.com/weaviate/typescript-client/releases/tag/v1.3.0) | 2023-05-08 |
- | [1.2.0](https://github.com/weaviate/typescript-client/releases/tag/v1.2.0) | 2023-05-05 |
- | [1.1.0](https://github.com/weaviate/typescript-client/releases/tag/v1.1.0) | 2023-04-21 |
- | [1.0.0](https://github.com/weaviate/typescript-client/releases/tag/v1.0.0) | 2023-05-04 |
-
+ Click here for a table of Weaviate and corresponding client versions
+import ReleaseHistory from '/_includes/release-history.md';
-import MatrixIntro from '/_includes/clients/matrix-intro.md';
+
-
+
## Change logs
diff --git a/developers/weaviate/concepts/index.md b/developers/weaviate/concepts/index.md
index 27bed2cf38..5be8160b74 100644
--- a/developers/weaviate/concepts/index.md
+++ b/developers/weaviate/concepts/index.md
@@ -1,5 +1,6 @@
---
title: Concepts
+description: Concepts in Weaviate
sidebar_position: 0
image: og/docs/concepts.jpg
# tags: ['getting started']
diff --git a/developers/weaviate/concepts/indexing.md b/developers/weaviate/concepts/indexing.md
index 0d40b229ed..dc288c9367 100644
--- a/developers/weaviate/concepts/indexing.md
+++ b/developers/weaviate/concepts/indexing.md
@@ -7,7 +7,7 @@ image: og/docs/concepts.jpg
Weaviate supports several types of indexes.
-1. **Vector indexes** - a vector index (e.g. HNSW or flat) is used to serve all vector-search queries.
+1. **[Vector indexes](./vector-index.md)** - a vector index (e.g. HNSW or flat) is used to serve all vector-search queries.
1. **Inverted indexes** - inverted indexes enable BM25 queries, or speed up filtering.
You can configure indexes in Weaviate per collection.
diff --git a/developers/weaviate/concepts/modules.md b/developers/weaviate/concepts/modules.md
index 1864e64f02..79d7125f7c 100644
--- a/developers/weaviate/concepts/modules.md
+++ b/developers/weaviate/concepts/modules.md
@@ -1,5 +1,6 @@
---
title: Modules
+description: Modules in Weaviate
sidebar_position: 15
image: og/docs/concepts.jpg
# tags: ['modules']
diff --git a/developers/weaviate/concepts/replication-architecture/consistency.md b/developers/weaviate/concepts/replication-architecture/consistency.md
index 4b7b8cb298..8ac241ad84 100644
--- a/developers/weaviate/concepts/replication-architecture/consistency.md
+++ b/developers/weaviate/concepts/replication-architecture/consistency.md
@@ -151,9 +151,61 @@ For speed, data operations on a tenant occur independently of any tenant activit
## Repairs
-When Weaviate detects inconsistent data across replicas, it attempts to repair the out of sync data.
+In distributed systems like Weaviate, object replicas can become inconsistent due to any number of reasons - network issues, node failures, or timing conflicts. When Weaviate detects inconsistent data across replicas, it attempts to repair the out of sync data.
-Starting in v1.26, Weaviate adds [async replication](#async-replication) to proactively detect inconsistencies. In earlier versions, Weaviate uses a [repair-on-read](#repair-on-read) strategy to repair inconsistencies at read time.
+Weaviate uses [async replication](#async-replication), [deletion resolution](#deletion-resolution-strategies) and [repair-on-read](#repair-on-read) strategies to maintain consistency across replicas.
+
+### Async replication
+
+:::info Added in `v1.26`
+:::
+
+Async replication runs in the background. It uses a Merkle tree algorithm to monitor and compare the state of nodes within a cluster. If the algorithm identifies an inconsistency, it resyncs the data on the inconsistent node.
+
+Repair-on-read works well with one or two isolated repairs. Async replication is effective in situations where there are many inconsistencies. For example, if an offline node misses a series of updates, async replication quickly restores consistency when the node returns to service.
+
+Async replication supplements the repair-on-read mechanism. If a node becomes inconsistent between sync checks, the repair-on-read mechanism catches the problem at read time.
+
+To activate async replication, set `asyncEnabled` to true in the [`replicationConfig` section of your collection definition](../../manage-data/collections.mdx#replication-settings).
+
+### Deletion resolution strategies
+
+:::info Added in `v1.28`
+:::
+
+When an object is present on some replicas but not others, this can be because a creation has not yet been propagated to all replicas, or because a deletion has not yet been propagated to all replicas. It is important to distinguish between these two cases.
+
+Deletion resolution works alongside async replication and repair-on-read to ensure consistent handling of deleted objects across the cluster. For each collection, you can set one of the following deletion resolution strategies:
+
+- `NoAutomatedResolution`
+- `DeleteOnConflict`
+- `TimeBasedResolution`
+
+#### `NoAutomatedResolution`
+
+This is the default setting, and the only setting available in Weaviate versions prior to `v1.28`. In this mode, Weaviate does not treat deletion conflicts as a special case. If an object is present on some replicas but not others, Weaviate may potentially restore the object on the replicas where it is missing.
+
+#### `DeleteOnConflict`
+
+A deletion conflict in `deleteOnConflict` is always resolved by deleting the object on all replicas.
+
+To do so, Weaviate updates an object as a deleted object on a replica upon receiving a deletion request, rather than removing all traces of the object.
+
+#### `TimeBasedResolution`
+
+A deletion conflict in `timeBasedResolution` is resolved based on the timestamp of the deletion request, in comparison to any subsequent updates to the object such as a creation or an update.
+
+If the deletion request has a timestamp that is later than the timestamp of any subsequent updates, the object is deleted on all replicas. If the deletion request has a timestamp that is earlier than the timestamp of any subsequent updates, the later updates are applied to all replicas.
+
+For example:
+- If an object is deleted at timestamp 100 and then recreated at timestamp 90, the recreation wins
+- If an object is deleted at timestamp 100 and then recreated at timestamp 110, the deletion wins
+
+#### Choosing a strategy
+
+- Use `NoAutomatedResolution` when you want maximum control and handle conflicts manually
+- Use `DeleteOnConflict` when you want to ensure deletions are always honored
+- Use `TimeBasedResolution` when you want the most recent operation to take precedence
### Repair-on-read
@@ -178,19 +230,6 @@ The read repair process also depends on the read and write consistency levels us
Repairs only happen on read, so they do not create a lot of background overhead. While nodes are in an inconsistent state, read operations with consistency level of `ONE` may return stale data.
-### Async replication
-
-:::info Added in `v1.26`
-:::
-
-Async replication runs in the background. It uses a Merkle tree algorithm to monitor and compare the state of nodes within a cluster. If the algorithm identifies an inconsistency, it resyncs the data on the inconsistent node.
-
-Repair-on-read works well with one or two isolated repairs. Async replication is effective in situations where there are many inconsistencies. For example, if an offline node misses a series of updates, async replication quickly restores consistency when the node returns to service.
-
-Async replication supplements the repair-on-read mechanism. If a node becomes inconsistent between sync checks, the repair-on-read mechanism catches the problem at read time.
-
-To activate async replication, set `asyncEnabled` to true in the [`replicationConfig` section of your collection definition](../../manage-data/collections.mdx#replication-settings).
-
## Related pages
- [API References | GraphQL | Get | Consistency Levels](../../api/graphql/get.md#consistency-levels)
- [API References | REST | Objects](/developers/weaviate/api/rest#tag/objects)
diff --git a/developers/weaviate/concepts/search/hybrid-search.md b/developers/weaviate/concepts/search/hybrid-search.md
new file mode 100644
index 0000000000..8c3f1ebbb2
--- /dev/null
+++ b/developers/weaviate/concepts/search/hybrid-search.md
@@ -0,0 +1,191 @@
+---
+title: Hybrid search
+sidebar_position: 60
+image: og/docs/concepts.jpg
+# tags: ['concepts', 'search', 'hybrid search', 'vector search', 'keyword search', 'bm25']
+---
+
+Hybrid search combines [vector search](./vector-search.md) and [keyword search (BM25)](./keyword-search.md) to leverage the strengths of both approaches. This takes into account results' semantic similarity (vector search) and exact keyword relevance (BM25), providing more comprehensive search results.
+
+A hybrid search runs both search types in parallel and combines their scores to produce a final ranking of results. This makes it versatile and robust, suitable for a wide range of search use cases.
+
+## How Hybrid Search Works
+
+In Weaviate, a hybrid search performs the following steps:
+
+1. Executes both searches in parallel:
+ - Vector search to find semantically similar content
+ - BM25 search to find keyword matches
+1. Combines the normalized scores using a [fusion method](#fusion-method)
+1. Returns results ranked by the combined scores
+
+```mermaid
+%%{init: {
+ 'theme': 'base',
+ 'themeVariables': {
+ 'primaryColor': '#4a5568',
+ 'primaryTextColor': '#2d3748',
+ 'primaryBorderColor': '#718096',
+ 'lineColor': '#718096',
+ 'secondaryColor': '#f7fafc',
+ 'tertiaryColor': '#edf2f7',
+ 'fontFamily': 'Inter, system-ui, sans-serif',
+ 'fontSize': '14px',
+ 'lineHeight': '1.4',
+ 'nodeBorder': '1px',
+ 'mainBkg': '#ffffff',
+ 'clusterBkg': '#f8fafc'
+ }
+}}%%
+
+flowchart LR
+ %% Style definitions
+ classDef systemBox fill:#f8fafc,stroke:#3182ce,stroke-width:1.5px,color:#2d3748,font-weight:bold
+ classDef processBox fill:#f8fafc,stroke:gray,stroke-width:0.5px,color:#2d3748,font-weight:bold
+ classDef component fill:white,stroke:#a0aec0,stroke-width:1px,color:#2d3748
+
+ %% Main flow
+ query["🔍 Query"] --> split["Query Processing"]
+
+ %% Parallel processes
+ split --> vector["Vector Search"]
+ split --> bm25["BM25 Search"]
+
+ %% Results combination
+ vector --> fusion["Score Fusion"]
+ bm25 --> fusion
+ fusion --> results["📑 Ranked Results"]
+
+ %% Parameters box
+ subgraph params["Search Parameters"]
+ alpha["Alpha: Balance between\nvector and keyword scores"]
+ fusion_type["Fusion Strategy:\nrankedFusion or relativeScoreFusion"]
+ end
+
+ params --> fusion
+
+ %% Apply styles
+ class query,split,vector,bm25,fusion,results component
+ class params processBox
+ class alpha,fusion_type component
+
+ %% Linkstyle for curved edges
+ linkStyle default stroke:#718096,stroke-width:3px,fill:none,background-color:white
+```
+
+### Fusion Strategies
+
+Weaviate supports two strategies (`relativeScoreFusion` and `rankedFusion`) for combining vector and keyword search scores:
+
+With `relativeScoreFusion` (default from `v1.24`), each object is scored by *normalizing* the metrics output by the vector search and keyword search respectively. The highest value becomes 1, the lowest value becomes 0, and others end up in between according to this scale. The total score is thus calculated by a scaled sum of normalized vector distance and normalized BM25 score.
+
+With `rankedFusion` (default for `v1.23` and lower), each object is scored according to its position in the results for the given search, starting from the highest score for the top-ranked object and decreasing down the order. The total score is calculated by adding these rank-based scores from the vector and keyword searches.
+
+Generally, `relativeScoreFusion` might be a a good choice, which is why it is the default.
+
+The main reason is that `relativeScoreFusion` retains more information from the original searches than `rankedFusion`, which only retains the rankings. More generally we believe that the nuances captured in the vector and keyword search metrics are more likely to be reflected in rankings produced by `relativeScoreFusion`.
+
+We include a concrete example of the two fusion strategies below.
+
+### Fusion Example
+
+Let's say that a search returns **five objects** with **document id** (from 0 to 4), and **scores** from **keyword** and **vector search**, **ordered by score**:
+
+
+
+
Search Type
+
(id): score
(id): score
(id): score
(id): score
(id): score
+
+
+
Keyword
+
(1): 5
(0): 2.6
(2): 2.3
(4): 0.2
(3): 0.09
+
+
+
Vector
+
(2): 0.6
(4): 0.598
(0): 0.596
(1): 0.594
(3): 0.009
+
+
+
+#### Ranked Fusion
+
+The score depends on the rank of each result and is computed according to `1/(RANK + 60)`, resulting in:
+
+
+
+
Search Type
+
(id): score
(id): score
(id): score
(id): score
(id): score
+
+
+
Keyword
+
(1): 0.0154
(0): 0.0160
(2): 0.0161
(4): 0.0167
(3): 0.0166
+
+
+
Vector
+
(2): 0.016502
(4): 0.016502
(0): 0.016503
(1): 0.016503
(3): 0.016666
+
+
+
+As you can see, the results for each rank are identical, regardless of the input score.
+
+#### Relative Score Fusion
+
+In relative score fusion, the largest score is set to 1 and the lowest to 0, and all entries in-between are scaled according to their **relative distance** to the **maximum** and **minimum values**.
+
+
+
+
Search Type
+
(id): score
(id): score
(id): score
(id): score
(id): score
+
+
+
Keyword
+
(1): 1.0
(0): 0.511
(2): 0.450
(4): 0.022
(3): 0.0
+
+
+
Vector
+
(2): 1.0
(4): 0.996
(0): 0.993
(1): 0.986
(3): 0.0
+
+
+
+The scores therefore reflect the relative distribution of the original scores. For example, the vector search scores of the first 4 documents were almost identical, which is still the case for the normalized scores.
+
+#### Comparison
+
+For the vector search, the scores for the top 4 objects (**IDs 2, 4, 0, 1**) were almost identical, and all of them were good results. While for the keyword search, one object (**ID 1**) was much better than the rest.
+
+This is captured in the final result of `relativeScoreFusion`, which identified the object **ID 1** the top result. This is justified because this document was the best result in the keyword search with a big gap to the next-best score and in the top group of vector search.
+
+In contrast, for `rankedFusion`, the object **ID 2** is the top result, closely followed by objects **ID 1** and **ID 0**.
+
+### Alpha Parameter
+
+The alpha value determines the weight of the vector search results in the final hybrid search results. The alpha value can range from 0 to 1:
+
+- `alpha = 0.5` (default): Equal weight to both searches
+- `alpha > 0.5`: More weight to vector search
+- `alpha < 0.5`: More weight to keyword search
+
+## Search Thresholds
+
+Hybrid search supports a maximum vector distance threshold through the `max vector distance` parameter.
+
+This threshold applies only to the vector search component of the hybrid search, allowing you to filter out results that are too dissimilar in vector space, regardless of their keyword search scores.
+
+For example, consider a maximum vector distance of `0.3`. This means objects with a vector distance higher than `0.3` will be excluded from the hybrid search results, even if they have high keyword search scores.
+
+This can be useful when you want to ensure semantic similarity meets a minimum standard while still taking advantage of keyword matching.
+
+There is no equivalent threshold parameter for the keyword (BM25) component of hybrid search or the final combined scores.
+
+This is because BM25 scores are not normalized or bounded like vector distances, making a universal threshold less meaningful.
+
+## Further resources
+
+- [How-to: Search](../../search/index.md)
+- [How-to: Hybrid search](../../search/hybrid.md)
+- [Blog: A deep dive into Weaviate's fusion algorithms](/blog/hybrid-search-fusion-algorithms)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/concepts/search/index.md b/developers/weaviate/concepts/search/index.md
index 3483b4950f..3cdd52dcf2 100644
--- a/developers/weaviate/concepts/search/index.md
+++ b/developers/weaviate/concepts/search/index.md
@@ -13,14 +13,67 @@ The following sections provide a conceptual overview of search in Weaviate, incl
## Search process
-The following table illustrates the search process in Weaviate. Around the core search process, there are several steps that can be taken to improve and manipulate the search results.
+The following table and figure illustrate the search process in Weaviate. Around the core search process, there are several steps that can be taken to improve and manipulate the search results.
| Step | Description | Optional |
|------|-------------|----------|
| 1. [Retrieval](#retrieval-filter) | [Filter](#retrieval-filter): Narrow result sets based on criteria [Search](#retrieval-search): Find the most relevant entries, using one of [keyword](#keyword-search), [vector](#vector-search) or [hybrid](#hybrid-search) search types | Required |
-| 2. [Reranking](#rerank) | Reorder results using a different (e.g. more complex) model | Optional |
+| 2. [Rerank](#rerank) | Reorder results using a different (e.g. more complex) model | Optional |
| 3. [Retrieval augmented generation](#retrieval-augmented-generation-rag) | Send retrieved data and a prompt to a generative AI model. Also called retrieval augmented generation, or RAG. | Optional |
+
+
Here is a brief overview of each step:
### Retrieval: Filter
@@ -139,9 +192,13 @@ For example:
+:::info Read more
+See the [keyword search](./keyword-search.md) page for more details on how keyword search works in Weaviate.
+:::
+
#### Vector Search
-Similarity-based search using [vector embeddings](#vector-embeddings). This method compares vector representations of the query against those of the stored objects to find the closest matches, based on a predefined [distance metric](../../config-refs/distances.md).
+Similarity-based search using vector embeddings. This method compares vector representations of the query against those of the stored objects to find the closest matches, based on a predefined [distance metric](../../config-refs/distances.md).
In Weaviate, you can perform vector searches in multiple ways. You can search for similar objects based on [a text input](../../search/similarity.md#search-with-text), [a vector input](../../search/similarity.md#search-with-a-vector), or [an exist object](../../search/similarity.md#search-with-an-existing-object). You can even search for similar objects with other modalities such as [with images](../../search/image.md).
@@ -187,9 +244,13 @@ For example:
+:::info Read more
+See the [vector search](./vector-search.md) page for more details on how vector search works in Weaviate.
+:::
+
#### Hybrid Search
-Combines vector and keyword search to leverage the strengths of both approaches. Both searches are carried out and the results are combined using the selected [hybrid fusion method](#hybrid-fusion-method) and the given [alpha](#alpha-hybrid-search) value.
+Combines vector and keyword search to leverage the strengths of both approaches. Both searches are carried out and the results are combined using the selected parameters, such as the hybrid fusion method and the alpha value.
Hybrid Search: Example
@@ -234,6 +295,10 @@ For example:
+:::info Read more
+See the [hybrid search](./hybrid-search.md) page for more details on how hybrid search works in Weaviate.
+:::
+
### Retrieval: Unordered
Queries can be formulated without any ranking mechanisms.
@@ -274,7 +339,7 @@ Retrieval augmented generation (RAG), also called generative search, combines se
Weaviate integrates with many popular [generative model providers](../../model-providers/index.md) such as [AWS](../../model-providers/aws/generative.md), [Cohere](../../model-providers/cohere/generative.md), [Google](../../model-providers/google/generative.md), [OpenAI](../../model-providers/openai/generative.md) and [Ollama](../../model-providers/ollama/generative.md).
-As a result, Weaviate makes RAG easy to [set up](../../manage-data/collections.mdx#specify-a-generative-module), and easy to [execute as an integrated, single query](../../search/generative.md#grouped-task-search).
+As a result, Weaviate makes RAG easy to [set up](../../manage-data/collections.mdx#specify-a-generative-model-integration), and easy to [execute as an integrated, single query](../../search/generative.md#grouped-task-search).
RAG: Example
@@ -307,31 +372,22 @@ The search results consist of `{"description": "black bear"}` and `{"description
```
-## Glossary
-
-#### Vector embeddings
-
-A vector embedding captures semantic meaning of an object in a vector space. It consists of a set of numbers that represent the object's features. Vector embeddings are generated by a vectorizer model, which is a machine learning model that is trained for this purpose.
-
-#### BM25F algorithm
-
-The BM25 algorithm ranks matching documents according to their relevance to a given search query. At a high level, the BM25 algorithm uses the count of query terms in the document (term frequency) against the overall frequency of the term in the dataset (inverse document frequency) to calculate a relevance score.
-
-The BM25F variant extends the BM25 algorithm to support multiple fields in the search index.
-
-#### Hybrid fusion method
-
-Weaviate uses a hybrid fusion method to combine the results of vector and keyword searches. The method determines how the results of the two searches are combined to produce the final search results.
+## Search scores and metrics
-There are two algorithms available: `relativeScoreFusion` (default from `1.24`) and `rankedFusion` (default until `1.24`).
+Weaviate uses a variety of metrics to rank search results of a given query. The following metrics are used in Weaviate:
-With `rankedFusion`, each object is scored according to its position in the results for the given search, starting from the highest score for the top-ranked object and decreasing down the order. The total score is calculated by adding these rank-based scores from the vector and keyword searches.
+- Vector distance: A vector distance measure between the query and the object.
+- BM25F score: A keyword search score calculated using the BM25F algorithm.
+- Hybrid score: A combined score from vector and keyword searches.
-With `relativeScoreFusion`, each object is scored by *normalizing* the metrics output by the vector search and keyword search respectively. The highest value becomes 1, the lowest value becomes 0, and others end up in between according to this scale. The total score is thus calculated by a scaled sum of normalized vector similarity and normalized BM25 score.
+## Further resources
-#### Alpha (hybrid search)
+For more details, see the respective pages for:
+- [Concepts: Vector search](./vector-search.md)
+- [Concepts: Keyword search](./keyword-search.md)
+- [Concepts: Hybrid search](./hybrid-search.md).
-The alpha value determines the weight of the vector search results in the final hybrid search results. The alpha value ranges from 0 to 1, where 0 means only keyword search results are considered, and 1 means only vector search results are considered.
+For code snippets on how to use these search types, see the [How-to: search](../search/index.md) page.
## Questions and feedback
diff --git a/developers/weaviate/concepts/search/keyword-search.md b/developers/weaviate/concepts/search/keyword-search.md
new file mode 100644
index 0000000000..a8f64e6822
--- /dev/null
+++ b/developers/weaviate/concepts/search/keyword-search.md
@@ -0,0 +1,182 @@
+---
+title: Keyword Search (BM25)
+sidebar_position: 40
+image: og/docs/concepts.jpg
+# tags: ['concepts', 'search', 'keyword search', 'bm25', 'keyword']
+---
+
+Keyword search is an exact matching-based search using "tokens", or strings of characters.
+
+It uses the BM25 algorithm, which ranks matching documents according to their relevance to a given search query. At a high level, the BM25 algorithm uses the count of query terms in the document (term frequency) against the overall frequency of the term in the dataset (inverse document frequency) to calculate a relevance score.
+
+More specifically, Weaviate uses the BM25F algorithm, which extends BM25 to support using multiple fields in the search index.
+
+A keyword search determines the best matches based on the matches of exact tokens contained in the query against those of the stored objects. As a result, a keyword search is a good choice when exact matches (e.g. exact domain-specific language, precise categories or tags) are important. For example:
+
+- Searching for documents containing specific technical terms
+- Identifying articles by precise keywords or tags
+
+This differs from vector search, which finds semantically similar content even when the exact words don't match. You might use keyword search when precision is more important than finding related concepts.
+
+## Keyword search in Weaviate
+
+In Weaviate, a keyword search will return the objects best matching the query, as measured by [BM25F](https://en.wikipedia.org/wiki/Okapi_BM25) "score".
+
+:::info BM25F vs BM25
+The "F" in BM25F stands for "field", indicating that it is a field-specific version of BM25. This allows for different weights for different fields, or properties, of the objects.
+
+
+In Weaviate, they are used interchangeably, as the BM25F algorithm is used to calculate the scores for keyword searches. Here we will refer to it generally as BM25.
+:::
+
+A BM25 score is calculated based on the frequency of the query tokens in the object properties, as well as the length of the object properties and the query.
+
+When an input string such as `"A red Nike shoe"` is provided as the query, Weaviate will:
+
+1. [Tokenize](#tokenization) the input (e.g. to `["a", "red", "nike", "shoe"]`)
+2. Remove any [stopwords](#stopwords) (e.g. remove `a`, to produce `["red", "nike", "shoe"]`)
+3. Determine the BM25 scores against [selected properties](#selected-properties) of the database objects, based on the [BM25 parameters](#bm25-parameters) and any [property boosting](#property-boosting).
+4. Return the objects with the highest BM25 scores as the search results
+
+### Tokenization
+
+Tokenization for keyword searches refers to how each source text is split up into individual "tokens" to be compared and matched.
+
+The default tokenization method is `word`.
+
+Other tokenization methods such as `whitespace`, `lowercase`, and `field` are available, as well as specialized ones such as `GSE` or `kagome_kr` for other languages ([more details](../../config-refs/schema/index.md#tokenization)).
+
+Set the tokenization option [in the inverted index configuration](../../search/bm25.md#set-tokenization) for a collection.
+
+:::info Tokenization in different contexts
+
+The term "tokenization" is used in other contexts such as vectorization, or language generation. Note that each of these typically use different tokenizers to meet different requirements. This results in different sets of tokens, even from the same input text.
+
+:::
+
+### Stopwords
+
+Stopwords are words that are filtered out before processing text.
+
+Weaviate uses configurable stopwords in calculating the BM25 score. Any tokens that are contained in the stopword list will be ignored from the BM25 score calculation.
+
+See the [reference page](../../config-refs/schema/index.md#stopwords-stopword-lists) for more details.
+
+### BM25 Parameters
+
+BM25 is a scoring function used to rank documents based on the query terms appearing in them. It has two main parameters that control its behavior:
+
+- `k1` (default: 1.2): Controls term frequency saturation. Higher values mean that multiple occurrences of a term continue to increase the score more
+- `b` (default: 0.75): Controls document length normalization. Values closer to 1 mean more normalization for document length
+
+```mermaid
+%%{init: {
+ 'theme': 'base',
+ 'themeVariables': {
+ 'primaryColor': '#4a5568',
+ 'primaryTextColor': '#2d3748',
+ 'primaryBorderColor': '#718096',
+ 'lineColor': '#718096',
+ 'secondaryColor': '#f7fafc',
+ 'tertiaryColor': '#edf2f7',
+ 'fontFamily': 'Inter, system-ui, sans-serif',
+ 'fontSize': '14px',
+ 'lineHeight': '1.4',
+ 'nodeBorder': '1px',
+ 'mainBkg': '#ffffff',
+ 'clusterBkg': '#f8fafc'
+ }
+}}%%
+
+flowchart LR
+ %% Style definitions
+ classDef systemBox fill:#f8fafc,stroke:#3182ce,stroke-width:1.5px,color:#2d3748,font-weight:bold
+ classDef processBox fill:#f8fafc,stroke:gray,stroke-width:0.5px,color:#2d3748,font-weight:bold
+ classDef component fill:white,stroke:#a0aec0,stroke-width:1px,color:#2d3748
+ classDef paramBox fill:white,stroke:#3182ce,stroke-width:1px,color:#2d3748
+
+ %% Main flow
+ query["🔍 Query Text"] --> tokenize["⚡ Tokenization"]
+ tokenize --> stopwords["🚫 Stopword\nRemoval"]
+ stopwords --> scoring["📊 BM25\nScoring"]
+
+ %% Parameters section
+ subgraph params["Parameter Configuration"]
+ direction TB
+ k1["k1: Term Frequency\nSaturation Control"]
+ b["b: Document Length\nNormalization"]
+ end
+
+ params --> scoring
+ scoring --> results["📑 Ranked\nResults"]
+
+ %% Apply styles
+ class query,tokenize,stopwords,scoring,results component
+ class params processBox
+ class k1,b paramBox
+
+ %% Linkstyle for curved edges
+ linkStyle default stroke:#718096,stroke-width:3px,fill:none,background-color:white
+```
+
+Set custom `k1` and `b` values [for a collection](../../manage-data/collections.mdx#set-inverted-index-parameters).
+
+
+
+### Selected properties
+
+A BM25 query can optionally specify which object properties are to be included in the score calculations.
+
+By default, all `text` properties are included in a BM25 calculation. There are two ways to vary this:
+
+- In the collection configuration, [set `indexSearchable` for a property to `false`](../../manage-data/collections.mdx#property-level-settings). This property will then be ignored in all BM25 searches.
+- [Specify which properties to search at query time](../../search/bm25.md#search-on-selected-properties-only). This will only apply for that particular query.
+
+### Property Boosting
+
+Property boosting allows a query apply different weights to different properties when calculating the final BM25 score.
+
+This is useful when certain properties are more important for search than others.
+
+For example, when searching an e-commerce catalog, you could boost the title property and its categories over the product description.
+
+
+
+[Set the property weights](../../search/bm25.md#use-weights-to-boost-properties) at query time.
+
+## Combining with Vector Search
+
+Keyword search can be combined with vector search in Weaviate to perform a hybrid search. This allows you to leverage both:
+- Exact matching capabilities of keyword search
+- Semantic understanding of vector search
+
+See [Hybrid Search](./hybrid-search.md) for more information.
+
+## Notes and Best Practices
+
+Here are some key considerations when using keyword search:
+
+1. **Tokenization Choice**
+ - Choose based on your data and search requirements. For example, use `word` tokenization for natural language text, but consider `field` for URLs or email addresses that need exact matching as a whole.
+ - For multilingual content, consider specialized tokenizers like `GSE` for Chinese/Japanese or `kagome_kr` for Korean
+ - Consider special characters and case sensitivity needs
+ - Test your tokenization choice with subsets of your data and queries to ensure it handles special characters and case sensitivity as expected. You could perform these experiments with vectorization disabled to save resources/costs, as the two processes are independent.
+
+2. **Performance Optimization**
+ - Index only the properties you need for search
+ - Consider combining keyword search with vector search (i.e. perform a [hybrid search](./hybrid-search.md)) as a starting point, especially where you cannot anticipate users' behavior
+
+3. **Query Optimization**
+ - Consider boosting properties that are more important for search (e.g. title, category) over others (e.g. description)
+ - Only modify `k1` and `b` values if you have a good reason to do so, as the defaults are generally well-suited for most use cases
+
+### Further resources
+
+- [How-to: Search](../../search/index.md)
+- [How-to: Keyword search](../../search/bm25.md)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/concepts/search/vector-search.md b/developers/weaviate/concepts/search/vector-search.md
new file mode 100644
index 0000000000..20142e8035
--- /dev/null
+++ b/developers/weaviate/concepts/search/vector-search.md
@@ -0,0 +1,290 @@
+---
+title: Vector Search
+sidebar_position: 20
+image: og/docs/concepts.jpg
+# tags: ['concepts', 'search', 'vector search', 'vector']
+---
+
+Vector search is a similarity-based search using vector embeddings, or vectors. Vector search also referred to as "semantic search" due to its ability to find semantically similar objects.
+
+A vector embedding captures semantic meaning of an object in a vector space. It consists of a set of numbers that represent the object's features. Vector embeddings are generated by a vectorizer model, which is a machine learning model that is trained for this purpose.
+
+A vector search compares [vectors of the stored objects](#object-vectors) against the [query vector(s)](#query-vectors) to find the closest matches, before returning the top `n` results.
+
+:::tip An introduction to vector search
+New to vector search? Check out our blog, ["Vector Search Explained"](/blog/vector-search-explained) for an introduction to vector search concepts and use cases.
+:::
+
+## Object vectors
+
+For vector search, each object must have representative vector embeddings.
+
+The model used to generate vectors is called a vectorizer model, or an embedding model.
+
+A user can populate Weaviate with objects and their vectors in one of two ways:
+
+- Use Weaviate's [vectorizer model provider integrations](#model-provider-integration) to generate vectors
+- [Provide vectors directly](#bring-your-own-vector) to Weaviate
+
+### Model provider integration
+
+Weaviate provides [first-party integrations with popular vectorizer model providers](../../model-providers/index.md) such as [Cohere](../../model-providers/cohere/index.md), [Ollama](../../model-providers/ollama/index.md), [OpenAI](../../model-providers/openai/index.md), and more.
+
+In this workflow, the user can [configure a vectorizer for a collection](../../manage-data/collections.mdx#specify-a-vectorizer) and Weaviate will automatically generate vectors as needed, such as when inserting objects or performing searches.
+
+```mermaid
+%%{init: {
+ 'theme': 'base',
+ 'themeVariables': {
+ 'primaryColor': '#4a5568',
+ 'primaryTextColor': '#2d3748',
+ 'primaryBorderColor': '#718096',
+ 'lineColor': '#718096',
+ 'secondaryColor': '#f7fafc',
+ 'tertiaryColor': '#edf2f7',
+ 'fontFamily': 'Inter, system-ui, sans-serif',
+ 'fontSize': '14px',
+ 'lineHeight': '1.4',
+ 'nodeBorder': '1px',
+ 'mainBkg': '#ffffff',
+ 'clusterBkg': '#f8fafc'
+ }
+}}%%
+
+flowchart LR
+ %% Style definitions
+ classDef systemBox fill:#f8fafc,stroke:#3182ce,stroke-width:1.5px,color:#2d3748,font-weight:bold
+ classDef weaviateBox fill:#f8fafc,stroke:gray,stroke-width:0.5px,color:#2d3748,font-weight:bold
+ classDef cloudBox fill:white,stroke:#48bb78,stroke-width:2px,color:#553c9a,font-weight:bold
+ classDef providerBox fill:#f8fafc,stroke:gray,stroke-width:0.5px,color:#2d3748,font-weight:bold
+ classDef component fill:white,stroke:#a0aec0,stroke-width:1px,color:#2d3748
+ classDef edgeLabel fill:white,stroke:#e2e8f0,stroke-width:1px,color:#4a5568
+
+ %% Provider section
+ subgraph provider["Model provider"]
+ inference["🤖 Inference API /\nLocal model"]
+ end
+
+ %% Weaviate section
+ subgraph weaviate["Weaviate"]
+ vectorizer["🔌 Model provider\nintegration"]
+ core["💾 Data &\nvector store"]
+ end
+
+ %% User System
+ subgraph user["🖥️ User System"]
+ data["📄 Data"]
+ end
+
+ %% Connections with curved edges
+ data --->|"1. Insert objects\n(no vector)"| core
+ core --->|"2. Request\nvector"| vectorizer
+ vectorizer --->|"3. Request\nvector"| inference
+ inference --->|"4. Vector"| vectorizer
+ vectorizer --->|"5. Vector"| core
+
+ %% Apply styles
+ class user systemBox
+ class weaviate weaviateBox
+ class cloud cloudBox
+ class provider providerBox
+ class data,core,vectorizer,inference component
+
+ %% Linkstyle for curved edges
+ linkStyle default stroke:#718096,stroke-width:3px,fill:none,background-color:white
+```
+
+This integration abstracts the vector generation process from the user, allowing the user to focus on building applications and performing searches without worrying about the vector generation process.
+
+:::info Vectorizer configuration is immutable
+
+Once it is set, the vectorizer cannot be changed for a collection. This ensures that the vectors are generated consistently and stay compatible. If you need to change the vectorizer, you must create a new collection with the desired vectorizer, and [migrate the data to the new collection](../../manage-data/migrate.mdx).
+
+:::
+
+#### Manual vectors when vectorizer is configured
+
+Even when a vectorizer model is configured for a collection, a user can still provide vectors directly when inserting objects or performing a query. In this case, Weaviate will use the provided vector instead of generating a new one.
+
+This is useful when the user already has vectors generated by the same model, such as when importing objects from another system. Re-using the same vectors will save time and resources, as Weaviate will not need to generate new vectors.
+
+### Bring your own vector
+
+A user can directly upload vectors to Weaviate when inserting objects. This is useful when the user already has vectors generated by a model, or if the user wants to use a specific vectorizer model that does not have an integration with Weaviate.
+
+```mermaid
+%%{init: {
+ 'theme': 'base',
+ 'themeVariables': {
+ 'primaryColor': '#4a5568',
+ 'primaryTextColor': '#2d3748',
+ 'primaryBorderColor': '#718096',
+ 'lineColor': '#718096',
+ 'secondaryColor': '#f7fafc',
+ 'tertiaryColor': '#edf2f7',
+ 'fontFamily': 'Inter, system-ui, sans-serif',
+ 'fontSize': '14px',
+ 'lineHeight': '1.4',
+ 'nodeBorder': '1px',
+ 'mainBkg': '#ffffff',
+ 'clusterBkg': '#f8fafc'
+ }
+}}%%
+
+flowchart LR
+ %% Style definitions
+ classDef systemBox fill:#f8fafc,stroke:#3182ce,stroke-width:1.5px,color:#2d3748,font-weight:bold
+ classDef weaviateBox fill:#f8fafc,stroke:gray,stroke-width:0.5px,color:#2d3748,font-weight:bold
+ classDef component fill:white,stroke:#a0aec0,stroke-width:1px,color:#2d3748
+ classDef edgeLabel fill:white,stroke:#e2e8f0,stroke-width:1px,color:#4a5568
+
+ %% Weaviate section
+ subgraph weaviate["Weaviate"]
+ core["💾 Data &\nvector store"]
+ end
+
+ %% User System
+ subgraph user["🖥️ User System"]
+ data["📄 Data"]
+ end
+
+ %% Connections with curved edges
+ data --->|"1. Insert objects\n(with vectors)"| core
+
+ %% Apply styles
+ class user systemBox
+ class weaviate weaviateBox
+ class cloud cloudBox
+ class provider providerBox
+ class data,core,vectorizer,inference component
+
+ %% Linkstyle for curved edges
+ linkStyle default stroke:#718096,stroke-width:3px,fill:none,background-color:white
+```
+
+In this workflow, the user has the flexibility to use any vectorizer model and process independently of Weaviate.
+
+If using your own model, we recommend explicitly setting the vectorizer as `none` in the vectorizer configuration, such that you do not accidentally generate incompatible vectors with Weaviate.
+
+### Multiple object vectors
+
+:::info Added in `v1.24`
+:::
+
+A collections can be configured to allow each object to be represented my multiple vectors.
+
+Each such vector works as its distinct vector space that is independent of each other, referred to as a "named vector".
+
+A named vector can be configured with a [vectorizer model integration](#model-provider-integration), and may be provided using the ["bring your own vector"](#bring-your-own-vector) integration.
+
+## Query vectors
+
+In Weaviate, you can specify the query vector using:
+
+- A query vector (called `nearVector`),
+- A query object (called `nearObject`),
+- A query text (called `nearText`), or
+- A query media (called `nearImage` or `nearVideo`).
+
+In each of these cases, the search will return the most similar objects to the query, based on the vector embeddings of the query and the stored objects. However, they differ in how the query vector is specified to Weaviate.
+
+### `nearVector`
+
+In a `nearVector` query, the user provides a vector directly to Weaviate. This vector is compared to the vectors of the stored objects to find the most similar objects.
+
+### `nearObject`
+
+In a `nearObject` query, the user provides an object ID to Weaviate. Weaviate retrieves the vector of the object and compares it to the vectors of the stored objects to find the most similar objects.
+
+### `nearText` (and `nearImage`, `nearVideo`)
+
+In a `nearText` query, the user provides an input text to Weaviate. Weaviate uses the specified vectorizer model to generate a vector for the text, and compares it to the vectors of the stored objects to find the most similar objects.
+
+As a result, a `nearText` query is only available for collections where a vectorizer model is configured.
+
+A `nearImage` or `nearVideo` query works similarly to a `nearText` query, but with an image or video input instead of text.
+
+## Multi-target vector search
+
+:::info Added in `v1.26`
+:::
+
+In a multi-target vector search, Weaviate performs multiple, concurrent, single-target vector searches.
+
+These searches will produce multiple sets of results, each with a vector distance score.
+
+Weaviat combines these result sets, using a ["join strategy"](#available-join-strategies) to produce final scores for each result.
+
+If an object is within the search limit or the distance threshold of any of the target vectors, it will be included in the search results.
+
+If an object does not contain vectors for any selected target vector, Weaviate ignores that object and does not include it in the search results.
+
+### Available join strategies.
+
+- **minimum** (*default*) Use the minimum of all vector distances.
+- **sum** Use the sum of the vector distances.
+- **average** Use the average of the vector distances.
+- **manual weights** Use the sum of weighted distances, where the weight is provided for each target vector.
+- **relative score** Use the sum of weighted normalized distances, where the weight is provided for each target vector.
+
+## Vector index and search
+
+Weaviate uses vector indexes to facilitate efficient vector searches. Like other types of indexes, a vector index organizes vector embeddings in a way that allows for fast retrieval while optimizing for other needs such as search quality (e.g. recall), search throughput, and resource use (e.g. memory).
+
+In Weaviate, multiple types of vector indexes are available such as `hnsw`, `flat` and `dynamic` indexes.
+
+Each [collection](../data.md#collections) or [tenant](../data.md#multi-tenancy) in Weaviate will have its own vector index. Additionally, each collection or tenant can have [multiple vector indexes](../data.md#multiple-vectors-named-vectors), each with different configurations.
+
+:::info
+Read more about:
+- [Collections](../data.md#collections)
+- [Multi-tenancy](../data.md#multi-tenancy)
+- [Vector indexes](../vector-index.md)
+- [Multiple named vectors](../data.md#multiple-vectors-named-vectors)
+:::
+
+### Distance metrics
+
+There are many ways to measure vector distances, such as cosine distance, dot product, and Euclidean distance. Weaviate supports a variety of these distance metrics, as listed on the [distance metrics](../../config-refs/distances.md) page. Each vectorizer model is trained with a specific distance metric, so it is important to use the same distance metric for search as was used for training the model.
+
+Weaviate uses cosine distance as the default distance metric for vector searches, as this is the typical distance metric for vectorizer models.
+
+:::tip Distance vs Similarity
+In a "distance", the lower the value, the closer the vectors are to each other. In a "similarity", or "certainty" score, the higher the value, the closer the vectors are to each other. Some metrics, such as cosine distance, can also be expressed as a similarity score. Others, such as Euclidean distance, are only expressable as a distance.
+:::
+
+## Notes and best practices
+
+All compatible vectors are similar to some degree search.
+
+This has two effects:
+
+1. There will always be some "top" search results regardless of relevance.
+1. The entire dataset is always returned.
+
+If you search a vector database containing vectors for colors "Red", "Crimson" and "LightCoral" with a query vector for "SkyBlue", the search will still return a result (e.g. "Red"), even if it is not semantically similar to the query. The search is simply returning the closest match, even if it is not a good match in the absolute sense.
+
+As a result, Weaviate provides multiple ways to limit the search results:
+
+- **Limit**: Specify the maximum number of results to return.
+ - If not provided, defaults to system-defined [`QUERY_DEFAULTS_LIMIT`](../../config-refs/env-vars.md#general) of 10.
+- **AutoCut**: Limit results based on discontinuities in result metrics such as vector distance or search score.
+- **Threshold**: Specify a minimum similarity score (e.g. maximum cosine distance) for the results.
+- **Apply filters**: Use [filters](../filtering.md) to exclude results based on other criteria, such as metadata or properties.
+
+Use a combination of these methods to ensure that the search results are meaningful and relevant to the user.
+
+Generally, start with a `limit` to a maximum number of results to provide to the user, and adjust the `threshold` such that irrelevant results are unlikely to be returned.
+
+This will cause the search to return up to the specified (`limit`) number of results, but only if they are above the specified (`threshold`) similarity score.
+
+### Further resources
+
+- [How-to: Search](../../search/index.md)
+- [How-to: Vector similarity search](../../search/similarity.md)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/concepts/vector-index.md b/developers/weaviate/concepts/vector-index.md
index 31b8aa4a2a..0ccdceb0e8 100644
--- a/developers/weaviate/concepts/vector-index.md
+++ b/developers/weaviate/concepts/vector-index.md
@@ -5,7 +5,7 @@ image: og/docs/concepts.jpg
# tags: ['vector index plugins']
---
-Vector indexing is a key component of vector databases. It can help to [significantly **increase the speed** of the search process of similarity search](https://weaviate.io/blog/why-is-vector-search-so-fast) with only a minimal tradeoff in search accuracy ([HNSW index](#hierarchical-navigable-small-world-hnsw-index)), or efficiently store many subsets of data in a small memory footprint ([flat index](#flat-index)). The [dynamic index](#dynamic-index) can even start off as a flat index and then dynamically switch to the HNSW index as it scales past a threshold.
+What is vector indexing? It's a key component of vector databases that helps to [significantly **increase the speed** of the search process of similarity search](https://weaviate.io/blog/vector-search-explained) with only a minimal tradeoff in search accuracy ([HNSW index](#hierarchical-navigable-small-world-hnsw-index)), or efficiently store many subsets of data in a small memory footprint ([flat index](#flat-index)). The [dynamic index](#dynamic-index) can even start off as a flat index and then dynamically switch to the HNSW index as it scales past a threshold.
Weaviate's vector-first storage system takes care of all storage operations with a vector index. Storing data in a vector-first manner not only allows for semantic or context-based search, but also makes it possible to store *very* large amounts of data without decreasing performance (assuming scaled well horizontally or having sufficient shards for the indexes).
@@ -20,9 +20,13 @@ Available starting in `v1.25`. This is an experimental feature. Use with caution
This page explains what vector indexes are, and what purpose they serve in the Weaviate vector database.
+:::info What is a vector index?
+In vector databases, a vector index is a data structure that organizes vector embeddings to enable efficient similarity search. Indexing vector databases properly is crucial for performance, and different index types serve different purposes - from the simple flat index to more sophisticated approaches like HNSW.
+:::
+
## Why do you need vector indexing?
-[Vector embeddings](https://weaviate.io/blog/vector-embeddings-explained) are a great way to represent meaning. Vectors embeddings are arrays of elements that can capture meaning from different data types, such as texts, images, videos, and other content. The number of elements are called dimensions. High dimension vectors capture more information, but they are harder to work with.
+[Vector embeddings](https://weaviate.io/blog/vector-embeddings-explained) are a great way to represent meaning. Understanding how to index a vector is crucial for working with vector databases effectively. Vectors embeddings are arrays of elements that can capture meaning from different data types, such as texts, images, videos, and other content. The number of elements are called dimensions. High dimension vectors capture more information, but they are harder to work with.
Vector databases make it easier to work with high dimensional vectors. Consider search; Vector databases efficiently measure semantic similarity between data objects. When you run a [similarity search](https://weaviate.io/developers/weaviate/search/similarity), a vector database like Weaviate uses a vectorized version of the query to find objects in the database that have vectors similar to the query vector.
@@ -41,9 +45,11 @@ Another way to think of this is how products are placed in a supermarket. You'd
:::tip
-You might be also interested in our blog post [Why is vector search to fast?](https://weaviate.io/blog/why-is-vector-search-so-fast).
+You might be also interested in our blog post [Vector search explained](https://weaviate.io/blog/vector-search-explained).
:::
+Let's explore how to index a vector using different approaches supported by Weaviate. The first method is the HNSW index.
+
## Hierarchical Navigable Small World (HNSW) index
**Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive.
@@ -143,14 +149,28 @@ Cleanup is an async process runs that rebuilds the HNSW graph after deletes and
Available starting in `v1.22`. This is an experimental feature. Use with caution.
:::
-Starting in Weaviate `1.22`, you can use asynchronous indexing by opting in.
+This feature relates to the vector index, specifically only to the HNSW index.
-Asynchronous indexing decouples object creation from vector index updates. Objects are created faster, and the vector index updates in the background. Asynchronous indexing is especially useful for importing large amounts of data.
+Asynchronous indexing can be enabled by opting in as follows:
+- Open-source users can do this by setting the `ASYNC_INDEXING` environment variable to `true`.
+- Weaviate Cloud users can do this by toggling the `Enable async indexing` switch in the Weaviate Cloud Console.
-While the vector index is updating, Weaviate can search a maximum of 100,000 un-indexed objects by brute force, that is, without using the vector index. This means that the search performance is slower until the vector index has been fully updated. Also, any additional new objects beyond the first 100,000 in the queue are not include in the search.
+With synchronous indexing, the vector index is updated in lockstep with the object store. Updating an HNSW index can be an expensive operation, especially as the size of the index grows. As a result, the indexing operation can be the bottleneck in the system, slowing down the time for user requests to be completed.
-:::tip
-You might be also interested in our blog post [Vamana vs. HNSW - Exploring ANN algorithms Part 1](https://weaviate.io/blog/ann-algorithms-vamana-vs-hnsw).
+When asynchronous indexing is enabled, all vector indexing operations go through a queue. This applies to not only batch imports, but also to single object imports, deletions, and updates.
+
+This means that the object store can be updated quickly to finish performing user requests while the vector index updates in the background. Asynchronous indexing is especially useful for importing large amounts of data.
+
+This means that there will be a short delay between object creation and the object being available for vector search using the HNSW index. The number of objects in the queue can be monitored per node [as shown here](../config-refs/nodes.md).
+
+:::info Changes in `v1.28`
+In Weaviate `v1.22` to `v1.27`, the async indexing feature only affected batch import operations, using an in-memory queue.
+
+
+Starting in `v1.28`, the async indexing feature has been expanded to include single object imports, deletions, and updates. Additionally, the in-memory queue has been replaced with a persistent, on-disk queue. This change allows for more robust handling of indexing operations, and improves performance though reduction of lock contention and memory usage.
+
+
+The use of an on-disk queue may result in a slight increase in disk usage, however this is expected to be a small percentage of the total disk usage.
:::
## Flat index
@@ -158,7 +178,7 @@ You might be also interested in our blog post [Vamana vs. HNSW - Exploring ANN a
:::info Added in `v1.23`
:::
-The **flat index** is a simple, lightweight index that is fast to build and has a very small memory footprint. This index type is a good choice for use cases where each end user (i.e. tenant) has their own, isolated, dataset, such as in a SaaS product for example, or a database of isolated record sets.
+The **flat index** is one of the fundamental ways to implement vector indexing in databases. As the name suggests, it's a simple, lightweight index that is fast to build and has a very small memory footprint. This index type is a good choice for use cases where each end user (i.e. tenant) has their own, isolated, dataset, such as in a SaaS product for example, or a database of isolated record sets.
As the name suggests, the flat index is a single layer of disk-backed data objects and thus a very small memory footprint. The flat index is a good choice for small collections, such as for multi-tenancy use cases.
diff --git a/developers/weaviate/concepts/vector-quantization.md b/developers/weaviate/concepts/vector-quantization.md
index de013a8550..5a06caae69 100644
--- a/developers/weaviate/concepts/vector-quantization.md
+++ b/developers/weaviate/concepts/vector-quantization.md
@@ -14,8 +14,13 @@ Weaviate currently offers two vector quantization techniques:
- [Scalar quantization (SQ)](#scalar-quantization)
## What is quantization?
+
In general, quantization techniques reduce the memory footprint by representing numbers with lower precision numbers, like rounding a number to the nearest integer. In neural networks, quantization reduces the values of the weights or activations of the model stored as a 32-bit floating-point number (4 bytes) to a lower precision number, such as an 8-bit integer (1 byte).
+### What is vector quantization?
+
+Vector quantization is a technique that reduces the memory footprint of vector embeddings. Vector embeddings have been typically represented as 32-bit floating-point numbers. Vector quantization techniques reduce the size of the vector embeddings by representing them as smaller numbers, such as 8-bit integers or binary numbers. Some quantization techniques also reduce the number of dimensions in the vector embeddings.
+
## Product quantization
[Product quantization](https://ieeexplore.ieee.org/document/5432202) is a multi-step quantization technique that is available for use with `hnsw` indexes in Weaivate.
diff --git a/developers/weaviate/config-refs/datatypes.md b/developers/weaviate/config-refs/datatypes.md
index de05d59bb5..6358e9243f 100644
--- a/developers/weaviate/config-refs/datatypes.md
+++ b/developers/weaviate/config-refs/datatypes.md
@@ -1,5 +1,6 @@
---
title: Data types
+description: Data Types in Weaviate Schema
sidebar_position: 20
image: og/docs/configuration.jpg
# tags: ['Data types']
diff --git a/developers/weaviate/config-refs/distances.md b/developers/weaviate/config-refs/distances.md
index a924162958..476f2672de 100644
--- a/developers/weaviate/config-refs/distances.md
+++ b/developers/weaviate/config-refs/distances.md
@@ -27,8 +27,8 @@ In all cases, larger distance values indicate lower similarity. Conversely, smal
| `cosine` | Cosine (angular) distance. [See note 1 below] | `1 - cosine_sim(a,b)` | `0 <= d <= 2` | `0`: identical vectors
`2`: Opposing vectors. |
| `dot` | A dot product-based indication of distance.
More precisely, the negative dot product. [See note 2 below] | `-dot(a,b)` | `-∞ < d < ∞` | `-3`: more similar than `-2`
`2`: more similar than `5` |
| `l2-squared` | The squared euclidean distance between two vectors. | `sum((a_i - b_i)^2)` | `0 <= d < ∞` | `0`: identical vectors |
-| `hamming` | Number of differences between vectors at each dimensions. | sum(|a_i != b_i|) | `0 <= d < ∞` | `0`: identical vectors |
-| `manhattan` | The distance between two vector dimensions measured along axes at right angles. | sum(|a_i - b_i|) | `0 <= d < dims` | `0`: identical vectors |
+| `hamming` | Number of differences between vectors at each dimensions. | sum(|a_i != b_i|) | `0 <= d < dims` | `0`: identical vectors |
+| `manhattan` | The distance between two vector dimensions measured along axes at right angles. | sum(|a_i - b_i|) | `0 <= d < ∞` | `0`: identical vectors |
diff --git a/developers/weaviate/config-refs/env-vars.md b/developers/weaviate/config-refs/env-vars.md
index 8678c1bc13..d9f3918fe1 100644
--- a/developers/weaviate/config-refs/env-vars.md
+++ b/developers/weaviate/config-refs/env-vars.md
@@ -17,7 +17,7 @@ All other values are interpreted as `false`.
| Variable | Description | Type | Example Value |
| --- | --- | --- | --- |
-| `ASYNC_INDEXING` | (Experimental as of `v1.22`.)
If set, Weaviate creates vector indexes asynchronously to the object creation process. This can be useful for importing large amounts of data. (default: `false`) | `boolean` | `false` |
+| `ASYNC_INDEXING` | (Experimental, added in `v1.22`.)
If set, Weaviate creates vector indexes asynchronously to the object creation process. This can be useful for importing large amounts of data. (default: `false`) | `boolean` | `false` |
| `AUTOSCHEMA_ENABLED` | Whether to infer the schema where necessary with the autoschema (default: `true`) | `boolean` | `true` |
| `RAFT_ENABLE_ONE_NODE_RECOVERY` | Enable running the single node recovery routine on restart. This is useful if the
default hostname has changed and a single node cluster believes there are supposed to be two nodes. | `boolean` | `false` |
@@ -29,12 +29,14 @@ default hostname has changed and a single node cluster believes there are suppos
| `ENABLE_API_BASED_MODULES` | Enable all API-based modules. (Experimental as of `v1.26.0`) | `boolean` | `true` |
| `ENABLE_MODULES` | Specify Weaviate modules to enable | `string - comma separated names` | `text2vec-openai,generative-openai` |
| `ENABLE_TOKENIZER_GSE` | Enable the [`GSE` tokenizer](../config-refs/schema/index.md#gse-and-trigram-tokenization-methods) for use | `boolean` | `true` |
+| `ENABLE_TOKENIZER_KAGOME_JA` | Enable the [`Kagome` tokenizer for Japanese](../config-refs/schema/index.md#kagome_ja-tokenization-method) for use (Experimental as of `v1.28.0`) | `boolean` | `true` |
| `ENABLE_TOKENIZER_KAGOME_KR` | Enable the [`Kagome` tokenizer for Korean](../config-refs/schema/index.md#kagome_kr-tokenization-method) for use (Experimental as of `v1.25.7`) | `boolean` | `true` |
| `GODEBUG` | Controls debugging variables within the runtime. [See official Go docs](https://pkg.go.dev/runtime). | `string - comma-separated list of name=val pairs` | `gctrace=1` |
| `GOMAXPROCS` | Set the maximum number of threads that can be executing simultaneously. If this value is set, it be respected by `LIMIT_RESOURCES`. | `string - number` | `NUMBER_OF_CPU_CORES` |
| `GOMEMLIMIT` | Set the memory limit for the Go runtime. A suggested value is between 90-80% of your total memory for Weaviate. The Go runtime tries to make sure that long-lived and temporary memory allocations do not exceed this value by making the garbage collector more aggressive as the memory usage approaches the limit. [Learn more about GOMEMLIMIT](/blog/gomemlimit-a-game-changer-for-high-memory-applications). | `string - memory limit in SI units` | `4096MiB` |
| `GO_PROFILING_DISABLE` | If `true`, disables Go profiling. Default: `false`. | `boolean` | `false` |
| `GO_PROFILING_PORT` | Sets the port for the Go profiler. Default: `6060` | `integer` | `6060` |
+| `GRPC_MAX_MESSAGE_SIZE` | Maximum gRPC message size in bytes. (Added in `v1.27.1`) Default: 10MB | `string - number` | `2000000000` |
| `LIMIT_RESOURCES` | If `true`, Weaviate will automatically attempt to auto-detect and limit the amount of resources (memory & threads) it uses to (0.8 * total memory) and (number of cores-1). It will override any `GOMEMLIMIT` values, however it will respect `GOMAXPROCS` values. | `boolean` | `false` |
| `LOG_FORMAT` | Set the Weaviate logging format
Default: Outputs log data to json. e.g.: `{"action":"startup","level":"debug","msg":"finished initializing modules","time":"2023-04-12T05:07:43Z"}` `text`: Outputs log data to a string. e.g. `time="2023-04-12T04:54:23Z" level=debug msg="finished initializing modules" action=startup` | `string` | |
| `LOG_LEVEL` | Sets the Weaviate logging level. Default: `info`
`panic`: Panic entries only. (new in `v1.24`) `fatal`: Fatal entries only. (new in `v1.24`) `error`: Error entries only. (new in `v1.24`) `warning`: Warning entries only. (new in `v1.24`) `info`: General operational entries. `debug`: Very verbose logging. `trace`: Even finer-grained informational events than `debug`. | `string` | |
@@ -58,6 +60,8 @@ default hostname has changed and a single node cluster believes there are suppos
| `TOMBSTONE_DELETION_MAX_PER_CYCLE` | Maximum number of tombstones to delete per cleanup cycle. Set this to limit cleanup cycles, as they are resource-intensive. As an example, set a maximum of 10000000 (10M) for a cluster with 300 million-object shards. Default: none | `string - int` (New in `v1.24.15` / `v1.25.2`) | `10000000` |
| `TOMBSTONE_DELETION_MIN_PER_CYCLE` | Minimum number of tombstones to delete per cleanup cycle. Set this to prevent triggering unnecessary cleanup cycles below a threshold. As an example, set a minimum of 1000000 (1M) for a cluster with 300 million-object shards. Default: 0 (New in `v1.24.15`, `v1.25.2`) | `string - int` | `100000` |
| `USE_GSE` | Enable the [`GSE` tokenizer](../config-refs/schema/index.md#gse-and-trigram-tokenization-methods) for use. (The same as `ENABLE_TOKENIZER_GSE`. We recommend using `ENABLE_TOKENIZER_GSE` for consistency in naming with other optional tokenizers.) | `boolean` | `true` |
+| `USE_INVERTED_SEARCHABLE` | (Experimental) Store searchable properties using a more efficient in-disk format, designed for the BlockMax WAND algorithm. Set as `true` together with `USE_BLOCKMAX_WAND` to enable BlockMax WAND at query time. Added in `v1.28`. (Default: `false`)
This format is experimental and may be subject to breaking changes in future versions. Expect to need to re-index your data again if you enable this feature.
Migrations are not supported with this feature enabled. Start with a fresh index.
| `boolean` | `true` |
+| `USE_BLOCKMAX_WAND` | (Experimental) Use BlockMax WAND algorithm for BM25 and hybrid searches. Recommend enabling together with `USE_INVERTED_SEARCHABLE` to get the full performance benefits. Added in `v1.28`. (Default: `false`)
BlockMax WAND scores on single and multiple property search may be different due to different IDF and property length normalization
| `boolean` | `true` |
## Module-specific
@@ -75,6 +79,13 @@ default hostname has changed and a single node cluster believes there are suppos
| `TRANSFORMERS_INFERENCE_API` | The endpoint where to reach the transformers module if enabled | `string` | `http://t2v-transformers:8080` |
| `USE_GOOGLE_AUTH` | Automatically look for Google Cloud credentials, and generate Vertex AI access tokens for Weaviate to use as needed ([read more](../model-providers/google/index.md)). (default: `false`) | `boolean` | `true` |
| `USE_SENTENCE_TRANSFORMERS_VECTORIZER` | (EXPERIMENTAL) Use the `sentence-transformer` vectorizer instead of the default vectorizer (from the `transformers` library). Applies to custom images only. | `boolean` | `true` |
+| `CLIP_WAIT_FOR_STARTUP` | If `true`, Weaviate waits for the `multi2vec-clip` module to start up before starting (default: `true`). | `boolean` | `true` |
+| `NER_WAIT_FOR_STARTUP` | If `true`, Weaviate waits for the `ner-transformers` module to start up before starting (default: `true`). (Available from `v1.25.27`, `v1.26.12`, `v1.27.7`) | `boolean` | `true` |
+| `QNA_WAIT_FOR_STARTUP` | If `true`, Weaviate waits for the `qna-transformers` module to start up before starting (default: `true`). (Available from `v1.25.27`, `v1.26.12`, `v1.27.7`) | `boolean` | `true` |
+| `RERANKER_WAIT_FOR_STARTUP` | If `true`, Weaviate waits for the `reranker-transformers` module to start up before starting (default: `true`). (Available from `v1.25.27`, `v1.26.12`, `v1.27.7`) | `boolean` | `true` |
+| `SUM_WAIT_FOR_STARTUP` | If `true`, Weaviate waits for the `sum-transformers` module to start up before starting (default: `true`). (Available from `v1.25.27`, `v1.26.12`, `v1.27.7`) | `boolean` | `true` |
+| `GPT4ALL_WAIT_FOR_STARTUP` | If `true`, Weaviate waits for the `text2vec-gpt4all` module to start up before starting (default: `true`). (Available from `v1.25.27`, `v1.26.12`, `v1.27.7`) | `boolean` | `true` |
+| `TRANSFORMERS_WAIT_FOR_STARTUP` | If `true`, Weaviate waits for the `text2vec-transformers` module to start up before starting (default: `true`). (Available from `v1.25.27`, `v1.26.12`, `v1.27.7`) | `boolean` | `true` |
## Authentication and authorization
diff --git a/developers/weaviate/config-refs/schema/_category_.json b/developers/weaviate/config-refs/schema/_category_.json
index 4a2475dad6..7d3e4d209c 100644
--- a/developers/weaviate/config-refs/schema/_category_.json
+++ b/developers/weaviate/config-refs/schema/_category_.json
@@ -1,4 +1,4 @@
{
- "label": "Collection schema",
+ "label": "Collection definition",
"position": 2
-}
\ No newline at end of file
+}
diff --git a/developers/weaviate/config-refs/schema/index.md b/developers/weaviate/config-refs/schema/index.md
index 4c90259e37..95dcb39754 100644
--- a/developers/weaviate/config-refs/schema/index.md
+++ b/developers/weaviate/config-refs/schema/index.md
@@ -1,5 +1,6 @@
---
title: Collection definition
+description: Schema Configuration in Weaviate
sidebar_position: 10
image: og/docs/configuration.jpg
# tags: ['Data types']
@@ -545,6 +546,25 @@ The `gse` tokenizer is not loaded by default to save resources. To use it, set t
- `"素早い茶色の狐が怠けた犬を飛び越えた"`: `["素早", "素早い", "早い", "茶色", "の", "狐", "が", "怠け", "けた", "犬", "を", "飛び", "飛び越え", "越え", "た", "素早い茶色の狐が怠けた犬を飛び越えた"]`
- `"すばやいちゃいろのきつねがなまけたいぬをとびこえた"`: `["すばや", "すばやい", "やい", "いち", "ちゃ", "ちゃい", "ちゃいろ", "いろ", "のき", "きつ", "きつね", "つね", "ねが", "がな", "なま", "なまけ", "まけ", "けた", "けたい", "たい", "いぬ", "を", "とび", "とびこえ", "こえ", "た", "すばやいちゃいろのきつねがなまけたいぬをとびこえた"]`
+### `kagome_ja` tokenization method
+
+:::caution Experimental feature
+Available starting in `v1.28.0`. This is an experimental feature. Use with caution.
+:::
+
+For Japanese text, `kagome_ja` tokenization method is also available. This uses the [`Kagome` tokenizer](https://github.com/ikawaha/kagome?tab=readme-ov-file) with a Japanese [MeCab IPA](https://github.com/ikawaha/kagome-dict/) dictionary to split the property text.
+
+The `kagome_ja` tokenizer is not loaded by default to save resources. To use it, set the environment variable `ENABLE_TOKENIZER_KAGOME_JA` to `true` on the Weaviate instance.
+
+`kagome_ja` tokenization examples:
+
+- `"春の夜の夢はうつつよりもかなしき 夏の夜の夢はうつつに似たり 秋の夜の夢はうつつを超え 冬の夜の夢は心に響く 山のあなたに小さな村が見える 川の音が静かに耳に届く 風が木々を通り抜ける音 星空の下、すべてが平和である"`:
+ - [`"春", "の", "夜", "の", "夢", "は", "うつつ", "より", "も", "かなしき", "\n\t", "夏", "の", "夜", "の", "夢", "は", "うつつ", "に", "似", "たり", "\n\t", "秋", "の", "夜", "の", "夢", "は", "うつつ", "を", "超え", "\n\t", "冬", "の", "夜", "の", "夢", "は", "心", "に", "響く", "\n\n\t", "山", "の", "あなた", "に", "小さな", "村", "が", "見える", "\n\t", "川", "の", "音", "が", "静か", "に", "耳", "に", "届く", "\n\t", "風", "が", "木々", "を", "通り抜ける", "音", "\n\t", "星空", "の", "下", "、", "すべて", "が", "平和", "で", "ある"`]
+- `"素早い茶色の狐が怠けた犬を飛び越えた"`:
+ - `["素早い", "茶色", "の", "狐", "が", "怠け", "た", "犬", "を", "飛び越え", "た"]`
+- `"すばやいちゃいろのきつねがなまけたいぬをとびこえた"`:
+ - `["すばやい", "ちゃ", "いろ", "の", "きつね", "が", "なまけ", "た", "いぬ", "を", "とびこえ", "た"]`
+
### `kagome_kr` tokenization method
:::caution Experimental feature
@@ -557,9 +577,12 @@ The `kagome_kr` tokenizer is not loaded by default to save resources. To use it,
`kagome_kr` tokenization examples:
-- `"아버지가방에들어가신다"`: `["아버지", "가", "방", "에", "들어가", "신다"]`
-- `"아버지가 방에 들어가신다"`: `["아버지", "가", "방", "에", "들어가", "신다"]`
-- `"결정하겠다"`: `["결정", "하", "겠", "다"]`
+- `"아버지가방에들어가신다"`:
+ - `["아버지", "가", "방", "에", "들어가", "신다"]`
+- `"아버지가 방에 들어가신다"`:
+ - `["아버지", "가", "방", "에", "들어가", "신다"]`
+- `"결정하겠다"`:
+ - `["결정", "하", "겠", "다"]`
### Inverted index types
diff --git a/developers/weaviate/config-refs/schema/multi-vector.md b/developers/weaviate/config-refs/schema/multi-vector.md
index dbed8095f5..2fc0d4e0e1 100644
--- a/developers/weaviate/config-refs/schema/multi-vector.md
+++ b/developers/weaviate/config-refs/schema/multi-vector.md
@@ -39,6 +39,23 @@ Named vector collections support hybrid search, but only for one vector at a tim
[Keyword search](/developers/weaviate/search/bm25) syntax does not change if a collection has named vectors.
+### Query multiple named vectors
+
+:::info Added in `v1.26`
+:::
+
+Where multiple named vectors are defined in a collection, you can query them in a single search. This is useful for comparing the similarity of an object to multiple named vectors.
+
+This is called a "multi-target vector search".
+
+In a multi-target vector search, you can specify:
+
+- The target vectors to search
+- The query(ies) to compare to the target vectors
+- The weights to apply to each distance (raw, or normalized) for each target vector
+
+Read more in [How-to: Multi-target vector search](../../search/multi-vector.md).
+
## Related pages
- [How-to: manage data](/developers/weaviate/manage-data/collections#define-multiple-named-vectors): Configure collections
diff --git a/developers/weaviate/config-refs/schema/vector-index.md b/developers/weaviate/config-refs/schema/vector-index.md
index 3d4ef591ef..43192ff504 100644
--- a/developers/weaviate/config-refs/schema/vector-index.md
+++ b/developers/weaviate/config-refs/schema/vector-index.md
@@ -5,7 +5,7 @@ image: og/docs/configuration.jpg
# tags: ['configuration', 'vector index']
---
-Vector indexes facilitate efficient, vector-first data storage and retrieval.
+[Vector indexes](../../concepts/vector-index.md) facilitate efficient, vector-first data storage and retrieval.
## HNSW indexes
@@ -237,8 +237,7 @@ services:
QUERY_MAXIMUM_RESULTS: 10000
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'text2vec-openai'
- ENABLE_MODULES: 'text2vec-cohere,text2vec-huggingface,text2vec-openai,text2vec-google,generative-cohere,generative-openai,generative-google'
+ ENABLE_API_BASED_MODULES: 'true'
CLUSTER_HOSTNAME: 'node1'
AUTOSCHEMA_ENABLED: 'false'
ASYNC_INDEXING: 'true'
diff --git a/developers/weaviate/configuration/authentication.md b/developers/weaviate/configuration/authentication.md
index 49bf937ebe..f123ab4353 100644
--- a/developers/weaviate/configuration/authentication.md
+++ b/developers/weaviate/configuration/authentication.md
@@ -5,34 +5,84 @@ image: og/docs/configuration.jpg
# tags: ['authentication']
---
-Weaviate offers an optional authentication scheme using API keys and OpenID Connect (OIDC), which can enable various [authorizations](authorization.md) levels.
+:::info Authentication and authorization
+Authentication and authorization are closely related concepts, and sometimes abbreviated as `AuthN` and `AuthZ`. Authentication (`AuthN`) is the process of verifying the identity of a user, while authorization (`AuthZ`) is the process of determining what permissions the user has.
+:::
+
+Weaviate controls access through user authentication via API keys or OpenID Connect (OIDC), with an option for anonymous access. Users can then be assigned different [authorization](./authorization.md) levels, as shown in the diagram below.
+
+```mermaid
+flowchart LR
+ %% Define main nodes
+ Request["Client\nRequest"]
+ AuthCheck{"AuthN\nEnabled?"}
+ AccessCheck{"Check\nAuthZ"}
+ Access["✅ Access\nGranted"]
+ Denied["❌ Access\nDenied"]
+
+ %% Define authentication method nodes
+ subgraph auth ["AuthN"]
+ direction LR
+ API["API Key"]
+ OIDC["OIDC"]
+ AuthResult{"Success?"}
+ end
+
+ %% Define connections
+ Request --> AuthCheck
+ AuthCheck -->|"No"| AccessCheck
+ AuthCheck -->|"Yes"| auth
+ API --> AuthResult
+ OIDC --> AuthResult
+ AuthResult -->|"Yes"| AccessCheck
+ AuthResult -->|"No"| Denied
+
+ AccessCheck -->|"Pass"| Access
+ AccessCheck -->|"Fail"| Denied
+
+ %% Style nodes
+ style Request fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style AuthCheck fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style AccessCheck fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style Access fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style Denied fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style API fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style OIDC fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style AuthResult fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+ %% Style subgraph
+ style auth fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+```
+
+For example, a user logging in with the API key `jane-secret` may be granted administrator permissions, while another user logging in with the API key `ian-secret` may be granted read-only permissions.
-When authentication is disabled, all anonymous requests will be granted access.
+In summary, Weaviate allows the following authentication methods:
-In this documentation, we cover all scenarios for your convenience:
-- [Configuring Weaviate and the client for API key use](#api-key)
-- [Configuring Weaviate and the client for OIDC](#oidc---a-systems-perspective)
-- [Configuring Weaviate for anonymous access](#anonymous-access)
+- API key
+- OpenID Connect (OIDC)
+- Anonymous access (no authentication, strongly discouraged except for development or evaluation)
Note that API key and OIDC authentication can be both enabled at the same time.
-:::tip We recommend starting with the API key
-For most use cases, the API key option offers a balance between security and ease of use. Give it a try first, unless you have specific requirements that necessitate a different approach.
-:::
+The way to configure authentication differs by your deployment method, depending on whether you are running Weaviate in Docker or Kubernetes. Below, we provide examples for both.
-:::info Using Kubernetes?
-See [this page](../installation/kubernetes.md#authentication-and-authorization) for how to set up `values.yaml` for authentication & authorization.
+:::info What about Weaviate Cloud (WCD)?
+For Weaviate Cloud (WCD) instances, authentication is pre-configured with OIDC and API key access. You can [authenticate against Weaviate](../connections/connect-cloud.mdx) with your WCD credentials using OIDC, or [with API keys](/developers/wcs/platform/manage-api-keys).
:::
-## WCD authentication
+## API Key Authentication
-[Weaviate Cloud (WCD)](https://console.weaviate.cloud/) instances are pre-configured for API key [authentication](/developers/wcs/connect.mdx).
+API key authentication is a simple and effective way to authenticate users. Each user is assigned a unique API key, which is used to authenticate the user.
-## API key
+Note that you can either:
+- Set one user for all API keys, or
+- Define one user per API key (the number of users must match the number of API keys)
-To configure Weaviate for API key-based authentication, add the following environment variables to your configuration file.
+Make sure all listed users are also configured in the authorization settings.
-An example `docker-compose.yml` file looks like this:
+### API keys: Docker
+
+API key authentication can be configured using environment variables. In Docker Compose, set them in the configuration file (`docker-compose.yml`) such as in the following example:
```yaml
services:
@@ -40,82 +90,73 @@ services:
...
environment:
...
+ # Disable anonymous access.
+ AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false'
+
# Enables API key authentication.
AUTHENTICATION_APIKEY_ENABLED: 'true'
- # List one or more keys, separated by commas. Each key corresponds to a specific user identity below.
- AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'jane-secret-key,ian-secret-key'
+ # List one or more keys in plaintext separated by commas. Each key corresponds to a specific user identity below.
+ AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'viewer-key,admin-key'
# List one or more user identities, separated by commas. Each identity corresponds to a specific key above.
- AUTHENTICATION_APIKEY_USERS: 'jane@doe.com,ian-smith'
+ AUTHENTICATION_APIKEY_USERS: 'viewer-user,admin-user'
```
-The example associates API keys and users.
+This configuration:
+- Disables anonymous access
+- Enables API key authentication
+- Defines plaintext API keys in `AUTHENTICATION_APIKEY_ALLOWED_KEYS`
+- Associates users with the API keys in `AUTHENTICATION_APIKEY_USERS`
-| API key | User|
-| :- | :- |
-| `jane-secret-key` | `jane@doe.com` |
-| `ian-secret-key` | `ian-smith` |
+These users can now be assigned permissions based on the authorization settings.
-:::info `n` APIKEY_ALLOWED_KEYS vs `n` APIKEY_USERS
-There are two options for configuring the number of keys and users:
-- Option 1: There is exactly one user specified and any number of keys (all keys will end up using the same user).
-- Option 2: The lengths match, then key `n` will map to user `n`.
-:::
+### API keys: Kubernetes
-These users' permissions will be determined by the [authorization](./authorization.md) settings. Below is one such example configuration.
+For Kubernetes deployments using Helm, API key authentication can be configured in the `values.yaml` file under the `authentication` section. Here's an example configuration:
```yaml
-services:
- weaviate:
- ...
- environment:
- ...
- AUTHORIZATION_ADMINLIST_ENABLED: 'true'
- AUTHORIZATION_ADMINLIST_USERS: 'jane@doe.com,john@doe.com'
- AUTHORIZATION_ADMINLIST_READONLY_USERS: 'ian-smith,roberta@doe.com'
+authentication:
+ anonymous_access:
+ # Disable anonymous access.
+ enabled: false
+
+ apikey:
+ # Enables API key authentication.
+ enabled: true
+
+ # List one or more keys in plaintext separated by commas. Each key corresponds to a specific user identity below.
+ allowed_keys:
+ - admin-key
+ - viewer-key
+
+ # List one or more user identities, separated by commas. Each identity corresponds to a specific key above.
+ users:
+ - admin-user
+ - viewer-user
```
-The example associates permissions with users.
-
-| User| User type | Permission |
-| :- | :- | :- |
-| `jane-secret-key` | Admin | Read, write |
-| `john@doe.com` | Admin | Read, write |
-| `ian-smith` | Read Only | Read |
-| `roberta@doe.com` | Read Only | Read |
+This configuration:
+- Disables anonymous access
+- Enables API key authentication
+- Defines plaintext API keys in `allowed_keys`
+- Associates users with the API keys in `users`
-In the example, `jane-secret-key` is an `admin` key, and `ian-secret-key` is a `read-only` key.
-
-:::note What about the other identities?
-You might notice that the authorization list includes `john@doe.com` and `roberta@doe.com`. Weaviate supports a combination of API key and OIDC-based authentication. Thus, the additional users might be OIDC users.
+:::warning Environment Variables Take Precedence
+If you configure API keys using environment variables, those settings will take precedence over the values in `values.yaml`. To use the Helm values configuration, ensure you have not set the corresponding environment variables.
:::
-### API key: Client-side usage
-
-import APIKeyUsage from '/_includes/clients/api-token-usage.mdx';
-
-
-
-The cURL command looks like this:
+For enhanced security in production environments, you can store API keys in Kubernetes secrets and reference them using environment variables instead of storing them as plaintext in the Helm values.
-```bash
-curl https://${WEAVIATE_INSTANCE_URL}/v1/meta -H "Authorization: Bearer ${WEAVIATE_API_KEY}" | jq
-```
+## OIDC Authentication
-import ClientLibraryUsage from '/_includes/clients/client-library-usage.mdx';
+OIDC authentication requires the resource (Weaviate) to validate tokens issued by an identity provider. The identity provider authenticates the user and issues tokens, which are then validated by Weaviate.
-
+In an example setup, a Weaviate instance acts as the resource, Weaviate Cloud (WCD) acts as the identity provider, and the Weaviate client acts on behalf of the user.
-## OIDC
-
-OIDC authentication involves three parties.
-
-1. A **user** who wants to access a resource.
-1. An **identity provider (a.k.a token issuer)** (e.g. Okta, Microsoft, or WCD) that authenticates the user and issues tokens.
-1. A **resource** (in this case, Weaviate) who validates the tokens to rely on the identity provider's authentication.
+Any "OpenID Connect" compatible token issuer that implements OpenID Connect Discovery is compatible with Weaviate.
-For example, a setup may involve a Weaviate instance as a resource, Weaviate Cloud (WCD) as an identity provider, and the Weaviate client acting on behalf of the user. This document attempts to provide some perspective from each one to help you use Weaviate with authentication.
+This document discusses how to configure Weaviate as the resource.
@@ -133,38 +174,11 @@ correct, all contents of the token are trusted, which authenticates the user bas
-## OIDC: Server-side
+### OIDC: Docker
-Any "OpenID Connect" compatible token issuer that implements OpenID Connect Discovery is compatible with Weaviate.
-
-### Configure Weaviate as the resource
-
-Configuring the OIDC token issuer is outside the scope of this document, but here are a few options as a starting point:
+To configure Weaviate for OIDC-based authentication, add the following environment variables to your configuration file.
-- For simple use-cases such as for a single user, you can use Weaviate Cloud (WCD) as the OIDC token issuer. To do so:
- - Make sure you have a WCD account (you can [sign up here](https://console.weaviate.cloud/)).
- - In the Docker Compose file (e.g. `docker-compose.yml`), specify:
- - `https://auth.wcs.api.weaviate.io/auth/realms/SeMI` as the issuer (in `AUTHENTICATION_OIDC_ISSUER`),
- - `wcs` as the client id (in `AUTHENTICATION_OIDC_CLIENT_ID`), and
- - enable the adminlist (`AUTHORIZATION_ADMINLIST_ENABLED: 'true'`) and add your WCD account email as the user (in `AUTHORIZATION_ADMINLIST_USERS`) .
- - `email` as the username claim (in `AUTHENTICATION_OIDC_USERNAME_CLAIM`).
-
-- If you need a more customizable setup you can use commercial OIDC providers like [Okta](https://www.okta.com/).
-- As another alternative, you can run your own OIDC token issuer server, which may be the most complex but also configurable solution. Popular open-source solutions include Java-based [Keycloak](https://www.keycloak.org/) and Golang-based [dex](https://github.com/dexidp/dex).
-
-:::info
-By default, Weaviate validates that the token includes a specified client id in the audience claim. If your token issuer does not support this feature, you can turn it off as outlined in the configuration section below.
-:::
-
-### Set configuration options
-
-To use OpenID Connect (OIDC), the **respective environment variables** must be correctly configured in the configuration yaml for Weaviate.
-
-:::info
-As of November 2022, we were aware of some differences in Microsoft Azure's OIDC implementation compared to others. If you are using Azure and experiencing difficulties, [this external blog post](https://xsreality.medium.com/making-azure-ad-oidc-compliant-5734b70c43ff) may be useful.
-:::
-
-The OIDC-related Docker Compose environment variables are shown below. For configuration details, see the inline-yaml comments:
+An example `docker-compose.yml` file looks like this:
```yaml
services:
@@ -209,17 +223,120 @@ services:
AUTHENTICATION_OIDC_SCOPES: ''
```
-#### Get the Weaviate OpenID endpoint
+:::info OIDC and Azure
+As of November 2022, we were aware of some differences in Microsoft Azure's OIDC implementation compared to others. If you are using Azure and experiencing difficulties, [this external blog post](https://xsreality.medium.com/making-azure-ad-oidc-compliant-5734b70c43ff) may be useful.
+:::
-If you have OIDC authentication enabled, you can obtain Weaviate's OIDC configuration from the following endpoint:
+### OIDC: Kubernetes
-```bash
-curl ${WEAVIATE_INSTANCE_URL}/v1/.well-known/openid-configuration
+For Kubernetes deployments using Helm, OIDC authentication can be configured in the `values.yaml` file under the `authentication` section. Here's an example configuration:
+
+```yaml
+authentication:
+ anonymous_access:
+ # Disable anonymous access.
+ enabled: false
+ oidc:
+ # enabled (optional - defaults to false) turns OIDC auth on. All other fields in
+ # this section will only be validated if enabled is set to true.
+ enabled: true
+
+ # issuer (required) tells weaviate how to discover the token issuer. This
+ # endpoint must implement the OpenID Connect Discovery spec, so that weaviate
+ # can retrieve the issuer's public key.
+ #
+ # The example URL below uses the path structure commonly found with keycloak
+ # where an example realm 'my-weaviate-usecase' was created. The exact
+ # path structure depends on the token issuer. See the token issuer's documentation
+ # about which endpoint implements OIDC Discovery.
+ issuer: 'http://my-token-issuer/auth/realms/my-weaviate-usecase'
+
+ # client_id (required unless skip_client_id_check is set to true) tells
+ # Weaviate to check for a particular OAuth 2.0 client_id in the audience claim.
+ # This is to prevent that a token which was signed by the correct issuer
+ # but never intended to be used with Weaviate can be used for authentication.
+ #
+ # For more information on what clients are in OAuth 2.0, see
+ # https://tools.ietf.org/html/rfc6749#section-1.1
+ client_id: 'my-weaviate-client'
+
+ # username_claim (required) tells Weaviate which claim in the token to use for extracting
+ # the username. The username will be passed to the authorization plugin.
+ username_claim: 'email'
+
+ # skip_client_id_check (optional, defaults to false) skips the client_id
+ # validation in the audience claim as outlined in the section above.
+ # Not recommended to set this option as it reduces security, only set this
+ # if your token issuer is unable to provide a correct audience claim
+ skip_client_id_check: 'false'
+
+ # scope (optional) these will be used by clients as default scopes for authentication
+ scopes: ''
+
+ # groups_claim: ''
```
-Edit ${WEAVIATE_INSTANCE_URL} to provide your instance URL.
+### Note: Configuring the OIDC token issuer
-## OIDC: Client-side
+Configuring the OIDC token issuer is outside the scope of this document, but here are a few options as a starting point:
+
+- For simple use-cases such as for a single user, you can use Weaviate Cloud (WCD) as the OIDC token issuer. To do so:
+ - Make sure you have a WCD account (you can [sign up here](https://console.weaviate.cloud/)).
+ - In the Docker Compose file (e.g. `docker-compose.yml`), specify:
+ - `https://auth.wcs.api.weaviate.io/auth/realms/SeMI` as the issuer (in `AUTHENTICATION_OIDC_ISSUER`),
+ - `wcs` as the client id (in `AUTHENTICATION_OIDC_CLIENT_ID`), and
+ - enable the adminlist (`AUTHORIZATION_ADMINLIST_ENABLED: 'true'`) and add your WCD account email as the user (in `AUTHORIZATION_ADMINLIST_USERS`) .
+ - `email` as the username claim (in `AUTHENTICATION_OIDC_USERNAME_CLAIM`).
+
+- If you need a more customizable setup you can use commercial OIDC providers like [Okta](https://www.okta.com/).
+- As another alternative, you can run your own OIDC token issuer server, which may be the most complex but also configurable solution. Popular open-source solutions include Java-based [Keycloak](https://www.keycloak.org/) and Golang-based [dex](https://github.com/dexidp/dex).
+
+:::info
+By default, Weaviate validates that the token includes a specified client id in the audience claim. If your token issuer does not support this feature, you can turn it off as outlined in the configuration section below.
+:::
+
+## Anonymous Access
+
+Weaviate can be configured to accept anonymous requests. This is strongly discouraged except for development or evaluation purposes.
+
+Users that send requests without explicit authentication are authenticated as `user: anonymous`.
+
+You can use the authorization plugin to specify which permissions to apply to this `anonymous` user. If anonymous access is disabled altogether, any request without an allowed authentication scheme returns `401 Unauthorized`.
+
+### Anonymous access: Docker
+
+To enable anonymous access in Docker Compose, add the following environment variable to your configuration file:
+
+```yaml
+services:
+ weaviate:
+ ...
+ environment:
+ ...
+ AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+```
+
+### Anonymous access: Kubernetes
+
+To enable anonymous access in Kubernetes, add the following configuration to your `values.yaml` file:
+
+```yaml
+authentication:
+ anonymous_access:
+ enabled: true
+```
+
+## Client-side Usage
+
+### API key
+
+import APIKeyUsage from '/_includes/clients/api-token-usage.mdx';
+
+
+
+We recommend using a client library to authenticate against Weaviate. See [How-to: Connect](../connections/index.mdx) pages for more information.
+
+### OIDC
The OIDC standard allows for many different methods *(flows)* of obtaining tokens. The appropriate method can vary depending on your situation, including configurations at the token issuer, and your requirements.
@@ -239,9 +356,6 @@ import OIDCExamples from '/_includes/code/connections/oidc-connect.mdx';
-
-
-
### Get and pass tokens manually
@@ -357,31 +471,10 @@ For example, the cURL command looks like this:
curl https://localhost:8080/v1/objects -H "Authorization: Bearer ${WEAVIATE_API_KEY}" | jq
```
-
-
-## Anonymous access
-By default, Weaviate is configured to accept requests without any authentication headers or parameters. Users that send requests without explicit authentication are authenticated as `user: anonymous`.
-
-You can use the authorization plugin to specify which
-permissions to apply to anonymous users. If anonymous access is disabled altogether,
-any request without an allowed authentication scheme returns `401 Unauthorized`.
-
-### Configuration
-Anonymous access can be enabled or disabled in the configuration yaml using the environment variable shown below:
-
-```yaml
-services:
- weaviate:
- ...
- environment:
- ...
- AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
-```
-
-### How to use
-
-Send REST requests to Weaviate without any additional authentication headers or parameters.
+## Further resources
+- [Configuration: Authorization and RBAC](./authorization.md)
+- [References: Environment variables / Authentication and Authorization](../config-refs/env-vars.md#authentication-and-authorization)
## Questions and feedback
diff --git a/developers/weaviate/configuration/authorization.md b/developers/weaviate/configuration/authorization.md
index 056b803379..962eaa6616 100644
--- a/developers/weaviate/configuration/authorization.md
+++ b/developers/weaviate/configuration/authorization.md
@@ -1,35 +1,129 @@
---
-title: Authorization
+title: Authorization & RBAC
sidebar_position: 35
image: og/docs/configuration.jpg
# tags: ['authorization']
---
-The authorization plugin allows Weaviate to provide differentiated access to users based on their [authentication](./authentication.md) status. Along with allowing or disallowing anonymous access, Weaviate can differentiate between a user who is in the admin list, or on the read-only list.
+:::info Authentication and authorization
+Authentication and authorization are closely related concepts, and sometimes abbreviated as `AuthN` and `AuthZ`. Authentication (`AuthN`) is the process of verifying the identity of a user, while authorization (`AuthZ`) is the process of determining what permissions the user has.
+:::
-## Admin list
+Weaviate provides differentiated access through [authorization](./authorization.md) levels, based on the user's [authentication](./authentication.md) status. A user can be granted admin permission, read-only permission, or no permission at all. From `v1.28.0`, Weaviate also supports Role-Based Access Control (RBAC) for more fine-grained control over user permissions.
+
+The following diagram illustrates the flow of a user request through the authentication and authorization process:
+
+```mermaid
+flowchart TB
+ User(["Authenticated User"]) --> AuthScheme{"Authorization\nScheme?"}
+
+ subgraph rbac ["RBAC Authorization"]
+ direction TB
+ AdminRole["Admin Role"]
+ ViewerRole["Viewer Role"]
+ CustomRole["Custom Roles"]
+
+ Perms1["Full Access\nAll Operations"]
+ Perms2["Read-only\nAccess"]
+ Perms3["Custom\nPermissions"]
+
+ AdminRole --> Perms1
+ ViewerRole --> Perms2
+ CustomRole --> Perms3
+ end
-The admin list relies on the configured `Authentication Schema` to correctly identify
-the user. On each request, a check against a pre-configured admin list is done.
-If the user is contained in this list, they get all permissions. If they aren't,
-they get none. It is not currently possible to assign only some rights to a specific user.
+ subgraph adminlist ["Admin List Authorization"]
+ direction TB
+ AdminUser["Admin Users"]
+ ReadOnly["Read-only Users"]
+ AnonUser["Anonymous Users\n(Optional)"]
-## Read-only list
+ AllPerms["Full Access\nAll Operations"]
+ ReadPerms["Read-only\nAccess"]
+
+ AdminUser --> AllPerms
+ ReadOnly --> ReadPerms
+ AnonUser -.->|"If enabled"| AllPerms
+ AnonUser -.->|"If enabled"| ReadPerms
+ end
+
+ subgraph undiffer ["Undifferentiated Access"]
+ AllAccess["Full Access\nAll Operations"]
+ end
+
+ AuthScheme -->|"RBAC"| rbac
+ AuthScheme -->|"Admin List"| adminlist
+ AuthScheme -->|"Undifferentiated"| undiffer
+
+ %% Style nodes
+ style User fill:#f9f9f9,stroke:#666
+ style AuthScheme fill:#f5f5f5,stroke:#666
+ style AnonUser fill:#f9f9f9,stroke:#666,stroke-dasharray: 5 5
+
+ %% Style subgraphs
+ style rbac fill:#e6f3ff,stroke:#4a90e2
+ style adminlist fill:#e6ffe6,stroke:#2ea44f
+ style undiffer fill:#fff0e6,stroke:#ff9933
+```
-Other than a list of admins, it is also possible to specify a list of read-only users.
-Those users have permissions on all `get` and `list` operations, but no other
-permissions.
+## Available authorization schemes
-If a subject is present on both the admin and read-only list, Weaviate will
-throw an error on startup due to the invalid configuration.
+The following authorization schemes are available in Weaviate:
-## Usage
+- [Role-Based Access Control (RBAC)](#role-based-access-control-rbac)
+- [Admin list](#admin-list)
+- [Undifferentiated access](#undifferentiated-access)
-:::info Using Kubernetes?
-See [this page](../installation/kubernetes.md#authentication-and-authorization) for how to set up `values.yaml` for authentication & authorization.
+In the Admin list authorization scheme, [anonymous users](#anonymous-users) can be granted permissions.
+
+The way to configure authorization differs by your deployment method, depending on whether you are running Weaviate in Docker or Kubernetes. Below, we provide examples for both.
+
+:::info What about Weaviate Cloud (WCD)?
+For Weaviate Cloud (WCD) instances, authorization is pre-configured with Admin list access. You can [authenticate against Weaviate](../connections/connect-cloud.mdx) with your WCD credentials using OIDC, or [with admin or read-only API keys](/developers/wcs/platform/manage-api-keys).
+
+
+RBAC access will be available in WCD in a future release.
+:::
+
+## Role-Based Access Control (RBAC)
+
+:::caution RBAC technical preview
+Role-based access control (RBAC) is added `v1.28` as a **technical preview**. This means that the feature is still under development and may change in future releases, including potential breaking changes. **We do not recommend using this feature in production environments at this time.**
+
+
+We appreciate [your feedback](https://forum.weaviate.io/) on this feature.
:::
-Configure the admin plugin in the configuration yaml like so:
+Role-based access control (RBAC) is a method of restricting access to resources based on the roles of users. In Weaviate, RBAC allows you to define [roles](./roles.md) and assign permissions to those roles. Users can then be assigned to roles, and inherit the permissions associated with those roles.
+
+Weaviate comes with a set of predefined roles. These roles are:
+
+- `admin`: The admin role has full access to all resources in Weaviate.
+- `viewer`: The viewer role has read-only access to all resources in Weaviate.
+
+They are assigned to users based on the Weaviate configuration file. Once a user is assigned a predefined role, their permissions are set accordingly. These roles cannot be modified, and these users cannot have additional roles assigned to them.
+
+For more information on roles and permissions, see [this page](./roles.md).
+
+:::tip At least one admin user required
+When using RBAC, you must specify at least one user with the build-in admin role. This user will have full permissions to perform all actions in Weaviate. Otherwise, Weaviate will not start.
+:::
+
+### Discussions: RBAC and performance
+
+RBAC is a powerful feature that allows you to define fine-grained access control policies. However, it can also have an impact on performance as each operation must be checked against the user's permissions.
+
+The exact performance impact will depend on your set up and use case. In our internal testing, the most significant performance impact was seen for object creation operations.
+
+We did not observe additional performance penalities for using custom roles over the built-in roles.
+
+Here are some tips to optimize performance when using RBAC:
+- Monitor object creation performance
+- Use a high availability (i.e. 3+ nodes) setup to distribute the load
+
+### RBAC: Docker
+
+RBAC authorization can be configured using environment variables. In Docker Compose, set them in the configuration file (`docker-compose.yml`) such as in the following example:
```yaml
services:
@@ -37,24 +131,233 @@ services:
...
environment:
...
+ # Example authentication configuration using API keys
+ # OIDC access can also be used with RBAC
+ AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false'
+ AUTHENTICATION_APIKEY_ENABLED: 'true'
+ AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'viewer-key,admin-key,other-key'
+ AUTHENTICATION_APIKEY_USERS: 'viewer-user,admin-user,other-user'
+
+ # Authorization configuration
+ # Enable RBAC
+ AUTHORIZATION_ENABLE_RBAC: 'true'
+
+ # Provide pre-configured roles to users
+ # This assumes that the relevant user has been authenticated and identified
+ #
+ # You MUST define at least one admin user
+ AUTHORIZATION_ADMIN_USERS: 'admin-user'
+ AUTHORIZATION_VIEWER_USERS: 'viewer-user'
+```
+
+This configuration:
+- Enables RBAC
+- Configures `admin-user` as a user with built-in admin permissions
+- Configures `viewer-user` as a user with built-in viewer permissions
+- Configures `other-user` as a user with no built-in permissions
+
+The `other-user` can now be assigned custom roles and permissions using the [RBAC Roles API](./roles.md).
+
+### RBAC: Kubernetes
+
+For Kubernetes deployments using Helm, API key authentication can be configured in the `values.yaml` file under the `authorization` section. Here's an example configuration:
+
+```yaml
+# Example authentication configuration using API keys
+authentication:
+ anonymous_access:
+ enabled: false
+ apikey:
+ enabled: true
+ allowed_keys:
+ - admin-key
+ - viewer-key
+ - other-key
+ users:
+ - admin-user
+ - viewer-user
+ - other-user
+
+# Authorization configuration
+authorization:
+ rbac:
+ # Enable RBAC
+ enabled: true
+ # Provide pre-configured roles to users
+ # This assumes that the relevant user has been authenticated and identified
+ #
+ # You MUST define at least one admin user
+ admins:
+ - admin-user
+ viewers:
+ - viewer-user
+```
+
+This configuration:
+- Enables RBAC
+- Configures `admin-user` as a user with built-in admin permissions
+- Configures `viewer-user` as a user with built-in viewer permissions
+- Configures `other-user` as a user with no built-in permissions
+
+The `other-user` can now be assigned custom roles and permissions using the [RBAC Roles API](./roles.md).
+
+## Admin list
+
+The "Admin list" authorization scheme allows you to specify a list of admin users with full permissions to perform all actions in Weaviate, and a list of read-only users with permissions to perform only read operations.
+
+These permissions cannot be customized or extended. For more fine-grained control over user permissions, use [RBAC](#role-based-access-control-rbac) instead.
+
+Admin list authorization scheme cannot be used in combination with RBAC.
+
+### Admin list: Docker
+
+Admin list authorization can be configured using environment variables. In Docker Compose, set them in the configuration file (`docker-compose.yml`) such as in the following example:
+
+```yaml
+services:
+ weaviate:
+ ...
+ environment:
+ ...
+ # Example authentication configuration using API keys
+ # OIDC access can also be used with RBAC
+ AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false'
+ AUTHENTICATION_APIKEY_ENABLED: 'true'
+ AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'viewer-key,admin-key,other-key'
+ AUTHENTICATION_APIKEY_USERS: 'viewer-user,admin-user,other-user'
+
+ # Authorization configuration
+ # Enable admin list
AUTHORIZATION_ADMINLIST_ENABLED: 'true'
- AUTHORIZATION_ADMINLIST_USERS: 'jane@doe.com,john@doe.com'
- AUTHORIZATION_ADMINLIST_READONLY_USERS: 'ian-smith,roberta@doe.com'
+
+ # Provide pre-configured roles to users
+ # This assumes that the relevant user has been authenticated and identified
+ #
+ # You MUST define at least one admin user
+ AUTHORIZATION_ADMINLIST_USERS: 'admin-user'
+ AUTHORIZATION_ADMINLIST_READONLY_USERS: 'viewer-user'
```
-The above would enable the plugin and make users `jane@doe.com` and
-`john@doe.com` admins. Additionally, users `ian-smith` and `roberta@doe.com` will have read-only permissions.
+This configuration:
+- Enables Admin list authorization
+- Configures `admin-user` as a user with built-in admin permissions
+- Configures `viewer-user` as a user with built-in viewer permissions
-:::note User identifier
-As shown in the above example, any string can be used to identify the user. This depends on what you configured in the authentication settings. For example, OIDC users may be identified by their email address by setting `AUTHENTICATION_OIDC_USERNAME_CLAIM` as `email`, whereas API key-authenticated users may be identified as a plain string such as `ian-smith`.
-:::
+Note that in this configuration, `other-user` has no permissions.
+
+### Admin list: Kubernetes
+
+For Kubernetes deployments using Helm, API key authentication can be configured in the `values.yaml` file under the `authorization` section. Here's an example configuration:
+
+```yaml
+# Example authentication configuration using API keys
+authentication:
+ anonymous_access:
+ enabled: false
+ apikey:
+ enabled: true
+ allowed_keys:
+ - admin-key
+ - viewer-key
+ - other-key
+ users:
+ - admin-user
+ - viewer-user
+ - other-user
+
+# Authorization configuration
+authorization:
+ admin_list:
+ # Enable admin list
+ enabled: true
+
+ # Provide pre-configured roles to users
+ # This assumes that the relevant user has been authenticated and identified
+ #
+ # You MUST define at least one admin user
+ users:
+ - admin-user
+ read_only_users:
+ - viewer-user
+```
+
+### Anonymous users
+
+Anonymous users are identified as `anonymous` in Weaviate. In the Admin list authorization scheme, you can apply permissions to anonymous users. The RBAC authorization scheme is not compatible with anonymous users.
+
+To confer permissions to anonymous users in the Admin list scheme, you can use the `anonymous` keyword in the configuration as shown below.
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+
+
+
+
+```yaml
+services:
+ weaviate:
+ ...
+ environment:
+ ...
+ # Enable anonymous access
+ AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+
+ # Configure admin user API key
+ AUTHORIZATION_ADMINLIST_ENABLED: 'true'
+ AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'admin-key'
+ AUTHENTICATION_APIKEY_USERS: 'admin-user'
+
+ # Enable admin list and provide admin access to "admin-user" only
+ AUTHORIZATION_ADMINLIST_USERS: 'admin-user'
+ # Provide read-only access to anonymous users
+ AUTHORIZATION_ADMINLIST_READONLY_USERS: 'anonymous'
+```
+
+
+
+
+
+```yaml
+# Example authentication configuration using API keys
+authentication:
+ # Enable anonymous access
+ anonymous_access:
+ enabled: true
+
+ # Enable admin list and configure admin user API key
+ apikey:
+ enabled: true
+ allowed_keys:
+ - admin-key
+ users:
+ - admin-user
+
+authorization:
+ # Enable admin list and provide admin access to "admin-user" only
+ admin_list:
+ # Enable admin list
+ enabled: true
+ users:
+ - admin-user
+ # Provide read-only access to anonymous users
+ read_only_users:
+ - anonymous
+```
+
+
+
+
+
+## Undifferentiated access
-## RBAC
+Weaviate can be configured to provide undifferentiated access, by disabling authentication for example and enabling anonymous access. This configuration is strongly discouraged except for development or evaluation purposes.
-More fine-grained Role-Based Access Control (RBAC) coming soon. As of now the
-only possible distinction is between Admins (CRUD), Read-Only Users and
-entirely unauthorized users.
+## Further resources
+- [Configuration: Authentication](./authentication.md)
+- [Configuration: RBAC Roles](./roles.md)
+- [References: Environment variables / Authentication and Authorization](../config-refs/env-vars.md#authentication-and-authorization)
## Questions and feedback
diff --git a/developers/weaviate/configuration/backups.md b/developers/weaviate/configuration/backups.md
index dfc0aee570..369830a877 100644
--- a/developers/weaviate/configuration/backups.md
+++ b/developers/weaviate/configuration/backups.md
@@ -20,42 +20,154 @@ import CurlCode from '!!raw-loader!/_includes/code/howto/configure.backups.sh';
Weaviate's Backup feature is designed to work natively with cloud technology. Most notably, it allows:
-* Seamless integration with widely-used cloud blob storage, such as AWS S3, GCS, or Azure
+* Seamless integration with widely-used cloud blob storage, such as AWS S3, GCS, or Azure Storage
* Backup and Restore between different storage providers
-* Single-command backup and restore from the REST API
+* Single-command backup and restore
* Choice of backing up an entire instance, or selected collections only
-* Zero downtime & minimal impact for your users when backups are running
-* Easy Migration to new environments
+* Easy migration to new environments
-:::note
-_Single node backup is available starting in Weaviate `v1.15`. Multi-node backups is available starting in `v1.16`_.
-:::
+:::caution Important backup considerations
-:::caution Backups do not include inactive or offloaded tenants
-Backups of [multi-tenant collections](../concepts/data.md#multi-tenancy) will only include `active` tenants, and not `inactive` or `offloaded` tenants. [Activate tenants](../manage-data/multi-tenancy.md#activate-tenant) before creating a backup to ensure all data is included.
+- **Version Requirements**: If you are running Weaviate `v1.23.12` or older, you must [update](../more-resources/migration/index.md) to `v1.23.13` or higher before restoring a backup to prevent data corruption.
+- **[Multi-tenancy](../concepts/data.md#multi-tenancy) limitations**: Backups will only include `active` tenants. `Inactive` or `offloaded` tenants in multi-tenant collections will not be included. Be sure to [activate](../manage-data/multi-tenancy.md#activate-tenant) any required tenants before creating a backup.
:::
+## Backup Quickstart
+
+Get started with Weaviate backups in minutes using the local filesystem, which is a great option for development and testing.
+
+### 1. Configure Weaviate
+
+Add these environment variables to your Weaviate configuration (e.g. Docker or Kubernetes configuration file):
+
+```yaml
+# Enable the filesystem backup module
+ENABLE_MODULES=backup-filesystem
+
+# Set backup location (e.g. within a Docker container or on a Kubernetes pod)
+BACKUP_FILESYSTEM_PATH=/var/lib/weaviate/backups
+```
+
+### 2. Start a backup
+
+Restart Weaviate to apply the new configuration. Then, you are ready to start a backup:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+That's all there is to getting started with backups in Weaviate. The backup will be stored in the specified location on the local filesystem.
+
+You can also:
+- [Restore the backup](#restore-backup) to a Weaviate instance
+- [Check the status](#asynchronous-status-checking) of the backup (if you did not wait for completion)
+- [Cancel the backup](#cancel-backup) if needed
+
+Note that local backups are not suitable for production environments. For production, use a cloud provider like S3, GCS, or Azure Storage.
+
+The following sections provide more details on how to configure and use backups in Weaviate.
+
## Configuration
-In order to perform backups, a backup provider module must be activated. Multiple backup providers can be active at the same time. Currently `backup-s3`, `backup-gcs`, `backup-azure`, and `backup-filesystem` modules are available for S3, GCS, Azure or filesystem backups respectively.
+Weaviate supports four backup storage options:
+
+| Provider | Module Name | Best For | Multi-Node Support |
+|----------|------------|-----------|-------------------|
+| AWS S3 | `backup-s3` | Production deployments, AWS environments | Yes |
+| Google Cloud Storage | `backup-gcs` | Production deployments, GCP environments | Yes |
+| Azure Storage | `backup-azure` | Production deployments, Azure environments | Yes |
+| Local Filesystem | `backup-filesystem` | Development, testing, single-node setups | No |
-As it is built on Weaviate's [module system](/developers/weaviate/configuration/modules.md), additional providers can be added in the future.
+To use any provider:
+1. Enable the module
+ - Add the module name to the `ENABLE_MODULES` environment variable
+ - On Weaviate Cloud instances, a relevant default module is enabled
+2. Configure the required modules
+ - Option 1: Set the necessary environment variables
+ - Option 2 (Kubernetes): Configure the [Helm chart values](#kubernetes-configuration)
-All service-discovery and authentication-related configuration is set using
-environment variables.
+Note multiple providers can be enabled simultaneously
### S3 (AWS or S3-compatible)
-Use the `backup-s3` module to enable backing up to and restoring from any S3-compatible blob storage. This includes AWS S3, and MinIO.
+- Works with AWS S3 and S3-compatible services (e.g., MinIO)
+- Supports multi-node deployments
+- Recommended for production use
-To enable the module, add its name to the `ENABLE_MODULES` environment variable. Modules are comma-separated. To enable the module along with the `text2vec-transformers` module for example, set:
+To configure `backup-s3`, you need to enable the module and provide the necessary configuration.
+
+#### Enable module
+
+Add `backup-s3` to the `ENABLE_MODULES` environment variable. For example, to enable the S3 module along with the `text2vec-cohere` module, set:
```
-ENABLE_MODULES=backup-s3,text2vec-transformers
+ENABLE_MODULES=backup-s3,text2vec-cohere
```
#### S3 Configuration (vendor-agnostic)
-In addition to enabling the module, you need to configure it using environment variables. This configuration applies to any S3-compatible backend.
+
+This configuration applies to any S3-compatible backend.
| Environment variable | Required | Description |
| --- | --- | --- |
@@ -66,7 +178,7 @@ In addition to enabling the module, you need to configure it using environment v
#### S3 Configuration (AWS-specific)
-You must provide Weaviate with AWS authentication details. You can choose between access-key or ARN-based authentication:
+For AWS, provide Weaviate with authentication details. You can choose between access-key or ARN-based authentication:
#### Option 1: With IAM and ARN roles
@@ -83,16 +195,20 @@ The backup module will first try to authenticate itself using AWS IAM. If the au
### GCS (Google Cloud Storage)
-Use the `backup-gcs` module to enable backing up to and restoring from any Google Cloud Storage bucket.
+- Works with Google Cloud Storage
+- Supports multi-node deployments
+- Recommended for production use
+
+To configure `backup-gcs`, you need to enable the module and provide the necessary configuration.
+
+#### Enable module
-To enable the module, add its name to the `ENABLE_MODULES` environment variable. Modules are comma-separated. To enable the module along with the `text2vec-transformers` module for example, set:
+Add `backup-gcs` to the `ENABLE_MODULES` environment variable. For example, to enable the S3 module along with the `text2vec-cohere` module, set:
```
-ENABLE_MODULES=backup-gcs,text2vec-transformers
+ENABLE_MODULES=backup-gcs,text2vec-cohere
```
-In addition to enabling the module, you need to configure it using environment variables. There are bucket-related variables, as well as credential-related variables.
-
#### GCS bucket-related variables
| Environment variable | Required | Description |
@@ -116,12 +232,18 @@ This makes it easy to use the same module in different setups. For example, you
### Azure Storage
-Use the `backup-azure` module to enable backing up to and restoring from any Microsoft Azure Storage container.
+- Works with Microsoft Azure Storage
+- Supports multi-node deployments
+- Recommended for production use
+
+To configure `backup-azure`, you need to enable the module and provide the necessary configuration.
+
+#### Enable module
-To enable the module, add its name to the `ENABLE_MODULES` environment variable. Modules are comma-separated. To enable the module along with the `text2vec-transformers` module for example, set:
+Add `backup-azure` to the `ENABLE_MODULES` environment variable. For example, to enable the Azure module along with the `text2vec-cohere` module, set:
```
-ENABLE_MODULES=backup-azure,text2vec-transformers
+ENABLE_MODULES=backup-azure,text2vec-cohere
```
In addition to enabling the module, you need to configure it using environment variables. There are container-related variables, as well as credential-related variables.
@@ -157,20 +279,22 @@ At least one of `AZURE_STORAGE_CONNECTION_STRING` or `AZURE_STORAGE_ACCOUNT` mus
### Filesystem
-:::caution `backup-filesystem` - limitations
-`backup-filesystem` is only compatible with single-node backups. Use `backup-gcs` or `backup-s3` if support for multi-node backups is needed.
+- Works with Google Cloud Storage
+- Supports single-node deployments only
+- Not recommended for production use
-The filesystem provider is not intended for production use, as its availability is directly tied to the node on which it operates.
-:::
+To configure `backup-filesystem`, you need to enable the module and provide the necessary configuration.
-Instead of backing up to a remote backend, you can also back up to the local filesystem. This may be helpful during development, for example to be able to quickly exchange setups, or to save a state from accidental future changes.
+#### Enable module
-To allow backups to the local filesystem, add `backup-filesystem` to the `ENABLE_MODULES` environment variable. Modules are comma-separated. To enable the module along with the `text2vec-transformers` module for example, set:
+Add `backup-filesystem` to the `ENABLE_MODULES` environment variable. For example, to enable the S3 module along with the `text2vec-cohere` module, set:
```
-ENABLE_MODULES=backup-filesystem,text2vec-transformers
+ENABLE_MODULES=backup-filesystem,text2vec-cohere
```
+#### Backup Configuration
+
In addition to enabling the module, you need to configure it using environment variables:
| Environment variable | Required | Description |
@@ -179,53 +303,28 @@ In addition to enabling the module, you need to configure it using environment v
### Other Backup Backends
-Weaviate uses its [module system](/developers/weaviate/configuration/modules.md) to decouple the backup orchestration from the remote backup storage backends. It is easy to add new providers and use them with the existing backup API. If you are missing your desired backup module, you can open a feature request or contribute it yourself. For either option, join our Slack community to have a quick chat with us on how to get started.
-
+If you are missing your desired backup module, you can open a feature request on the [Weaviate GitHub repository](https://github.com/weaviate/weaviate/issues). We are also open to community contributions for new backup modules.
## API
-### Create Backup
-
-Once the modules are enabled and the configuration is provided, you can start a backup on any running instance with a single HTTP request.
-
-#### Method and URL
-
-```js
-POST /v1/backups/{backend}
-```
+For REST API documentation, see the [Backups section](https://weaviate.io/developers/weaviate/api/rest#tag/backups).
-#### Parameters
-
-##### URL Parameters
-
-| Name | Type | Required | Description |
-| ---- | ---- | ---- | ---- |
-| `backend` | string | yes | The name of the backup provider module without the `backup-` prefix, for example `s3`, `gcs`, `azure`, or `filesystem`. |
+### Create Backup
-##### Request Body
+Once the modules are enabled and the configuration is provided, you can start a backup on any running instance with a single request.
-The request takes a JSON object with the following properties:
+You can choose to include or exclude specific collections in the backup. If you do not specify any collections, all collections are included by default.
-| Name | Type | Required | Description |
-| ---- | ---- | ---- | ---- |
-| `id` | string (lowercase letters, numbers, underscore, minus) | yes | The id of the backup. This string must be provided on all future requests, such as status checking or restoration. |
-| `include` | list of strings | no | An optional list of collection names to be included in the backup. If not set, all collections are included. |
-| `exclude` | list of strings | no | An optional list of collection names to be excluded from the backup. If not set, no collections are excluded. |
-| `config` | object | no | An optional object to configure the backup. If not set, it will assign defaults from config table.|
+The `include` and `exclude` options are mutually exclusive. You can set none or exactly one of those.
-:::note
-You cannot set `include` and `exclude` at the same time. Set none or exactly one of those.
-:::
+##### Available `config` object properties
-##### Config object properties
| name | type | required | default | description |
| ---- | ---- | ---- | ---- |---- |
| `cpuPercentage` | number | no | `50%` | An optional integer to set the desired CPU core utilization ranging from 1%-80%. |
| `chunkSize` | number | no | `128MB` | An optional integer represents the desired size for chunks. Weaviate will attempt to come close the specified size, with a minimum of 2MB, default of 128MB, and a maximum of 512MB.|
-| `compressionLevel`| string | no | `DefaultCompression` | An optional compression level used by compression algorithm from options. `DefaultCompression`, `BestSpeed`, `BestCompression`|
-:::note
-Weaviate uses [gzip compression](https://pkg.go.dev/compress/gzip#pkg-constants) by default.
-:::
+| `compressionLevel`| string | no | `DefaultCompression` | An optional compression level used by compression algorithm from options. (`DefaultCompression`, `BestSpeed`, `BestCompression`) Weaviate uses [gzip compression](https://pkg.go.dev/compress/gzip#pkg-constants) by default. |
+
@@ -259,7 +358,7 @@ Weaviate uses [gzip compression](https://pkg.go.dev/compress/gzip#pkg-constants)
text={TSCodeLegacy}
startMarker="// START CreateBackup"
endMarker="// END CreateBackup"
- language="ts"
+ language="tsv2"
/>
@@ -330,7 +429,7 @@ The response contains a `"status"` field. If the status is `SUCCESS`, the backup
text={PyCodeV3}
startMarker="# START StatusCreateBackup"
endMarker="# END StatusCreateBackup"
- language="py"
+ language="pyv3"
/>
@@ -348,7 +447,7 @@ The response contains a `"status"` field. If the status is `SUCCESS`, the backup
text={TSCodeLegacy}
startMarker="// START StatusCreateBackup"
endMarker="// END StatusCreateBackup"
- language="ts"
+ language="tsv2"
/>
@@ -394,50 +493,32 @@ An ongoing backup can be cancelled at any time. The backup process will be stopp
/>
-
- ```ts
- // Coming soon
- ```
-
+
This operation is particularly useful if you have started a backup by accident, or if you would like to stop a backup that is taking too long.
### Restore Backup
-You can restore any backup to any machine as long as the name and number of nodes between source and target are identical. The backup does not need to be created on the same instance. Once a backup backend is configured, you can restore a backup with a single HTTP request.
-Note that a restore fails if any of the collections already exist on this instance.
-
-#### Method and URL
-
-```js
-POST /v1/backups/{backend}/{backup_id}/restore
-```
+You can restore any backup to any machine as long as the name and number of nodes between source and target are identical. The backup does not need to be created on the same instance. Once a backup backend is configured, you can restore a backup with a single request.
-#### Parameters
-
-##### URL Parameters
+As with backup creation, the `include` and `exclude` options are mutually exclusive. You can set none or exactly one of those. In a restore operation, `include` and `exclude` are relative to the collections contained in the backup. The restore process is not aware of collections that existed on the source machine if they were not part of the backup.
-| Name | Type | Required | Description |
-| ---- | ---- | ---- | ---- |
-| `backend` | string | yes | The name of the backup provider module without the `backup-` prefix, for example `s3`, `gcs`, or `filesystem`. |
-| `backup_id` | string | yes | The user-provided backup identifier that was used when sending the request to create the backup. |
-
-##### Request Body
-The request takes a json object with the following properties:
-
-| Name | Type | Required | Description |
-| ---- | ---- | ---- | ---- |
-| `include` | list of strings | no | An optional list of collection names to be included in the backup. If not set, all collections are included. |
-| `exclude` | list of strings | no | An optional list of collection names to be excluded from the backup. If not set, no collections are excluded. |
-| `config` | object | no | An optional object to configure the restore. If not set, it will assign defaults from config table.|
+Note that a restore fails if any of the collections already exist on this instance.
-*Note 1: You cannot set `include` and `exclude` at the same time. Set none or exactly one of those.*
+:::caution Restoring backups from `v1.23.12` and older
+If you are running Weaviate `v1.23.12` or older, first **[update Weaviate](../more-resources/migration/index.md) to version 1.23.13** or higher before restoring a backup.
+Versions prior to `v1.23.13` had a bug that could lead to data not being stored correctly from a backup of your data.
+:::
-*Note 2: `include` and `exclude` are relative to the collections contained in the backup. The restore process does not know which collections existed on the source machine if they were not part of the backup.*
+##### Available `config` object properties
-##### Config object properties
| name | type | required | default | description |
| ---- | ---- | ---- | ---- |---- |
| `cpuPercentage` | number | no | `50%` | An optional integer to set the desired CPU core utilization ranging from 1%-80%. |
@@ -456,7 +537,7 @@ The request takes a json object with the following properties:
text={PyCodeV3}
startMarker="# START RestoreBackup"
endMarker="# END RestoreBackup"
- language="py"
+ language="pyv3"
/>
@@ -474,7 +555,7 @@ The request takes a json object with the following properties:
text={TSCodeLegacy}
startMarker="// START RestoreBackup"
endMarker="// END RestoreBackup"
- language="ts"
+ language="tsv2"
/>
@@ -513,19 +594,6 @@ All client implementations have a "wait for completion" option which will poll t
If you set the "wait for completion" option to false, you can also check the status yourself using the Backup Restore Status API.
-```js
-GET /v1/backups/{backend}/{backup_id}/restore
-```
-
-#### Parameters
-
-##### URL Parameters
-
-| Name | Type | Required | Description |
-| ---- | ---- | ---- | ---- |
-| `backend` | string | yes | The name of the backup provider module without the `backup-` prefix, for example `s3`, `gcs`, or `filesystem`. |
-| `backup_id` | string | yes | The user-provided backup identifier that was used when sending the requests to create and restore the backup. |
-
The response contains a `"status"` field. If the status is `SUCCESS`, the restore is complete. If the status is `FAILED`, an additional error is provided.
@@ -542,7 +610,7 @@ The response contains a `"status"` field. If the status is `SUCCESS`, the restor
text={PyCodeV3}
startMarker="# START StatusRestoreBackup"
endMarker="# END StatusRestoreBackup"
- language="py"
+ language="pyv3"
/>
@@ -560,7 +628,7 @@ The response contains a `"status"` field. If the status is `SUCCESS`, the restor
text={TSCodeLegacy}
startMarker="// START StatusRestoreBackup"
endMarker="// END StatusRestoreBackup"
- language="ts"
+ language="tsv2"
/>
@@ -592,6 +660,13 @@ The response contains a `"status"` field. If the status is `SUCCESS`, the restor
+## Kubernetes configuration
+
+When running Weaviate on Kubernetes, you can configure the backup provider using Helm chart values.
+
+These values are available under the `backups` key in the `values.yaml` file. Refer to the inline documentation in the `values.yaml` file for more information.
+
+
## Technical Considerations
@@ -599,7 +674,7 @@ The response contains a `"status"` field. If the status is `SUCCESS`, the restor
The backup process is designed to be minimally invasive to a running setup. Even on very large setups, where terabytes of data need to be copied, Weaviate stays fully usable during backup. It even accepts write requests while a backup process is running. This sections explains how backups work under the hood and why Weaviate can safely accept writes while a backup is copied.
-Weaviate uses a custom [LSM Store](../concepts/storage.md#object-and-inverted-index-store) for it's object store and inverted index. LSM stores are a hybrid of immutable disk segments and an in-memory structure called a memtable that accepts all writes (including updates and deletes). Most of the time, files on disk are immutable, there are only three situations where files are changed:
+Weaviate uses a custom [LSM Store](../concepts/storage.md#object-and-inverted-index-store) for its object store and inverted index. LSM stores are a hybrid of immutable disk segments and an in-memory structure called a memtable that accepts all writes (including updates and deletes). Most of the time, files on disk are immutable, there are only three situations where files are changed:
1. Anytime a memtable is flushed. This creates a new segment. Existing segments are not changed.
2. Any write into the memtable is also written into a Write-Ahead-Log (WAL). The WAL is only needed for disaster-recovery. Once a segment has been orderly flushed, the WAL can be discarded.
@@ -629,6 +704,12 @@ The flexibility around backup providers opens up new use cases. Besides using th
For example, consider the following situation: You would like to do a load test on production data. If you would do the load test in production it might affect users. An easy way to get meaningful results without affecting uses it to duplicate your entire environment. Once the new production-like "loadtest" environment is up, create a backup from your production environment and restore it into your "loadtest" environment. This even works if the production environment is running on a completely different cloud provider than the new environment.
+## Troubleshooting and notes
+
+- Single node backup is available starting in Weaviate `v1.15`. Multi-node backups is available starting in `v1.16`.
+- In some cases, backups can take a long time, or get "stuck", causing Weaviate to be unresponsive. If this happens, you can [cancel the backup](#cancel-backup) and try again.
+- If a backup module is misconfigured, such as having an invalid backup path, it can cause Weaviate to not start. Review the system logs for any errors.
+
## Related pages
- [References: REST API: Backups](/developers/weaviate/api/rest#tag/backups)
diff --git a/developers/weaviate/configuration/compression/bq-compression.md b/developers/weaviate/configuration/compression/bq-compression.md
index 21c0d6a01b..ba70bbb98c 100644
--- a/developers/weaviate/configuration/compression/bq-compression.md
+++ b/developers/weaviate/configuration/compression/bq-compression.md
@@ -53,7 +53,7 @@ This can be done by setting the `vector_index_config` of the collection to enabl
text={PyCodeV3}
startMarker="# START EnableBQ"
endMarker="# END EnableBQ"
- language="py"
+ language="pyv3"
/>
@@ -71,7 +71,7 @@ This can be done by setting the `vector_index_config` of the collection to enabl
text={TSCodeLegacy}
startMarker="// START EnableBQ"
endMarker="// END EnableBQ"
- language="ts"
+ language="tsv2"
/>
@@ -124,7 +124,7 @@ For example:
text={PyCodeV3}
startMarker="# START BQWithOptions"
endMarker="# END BQWithOptions"
- language="py"
+ language="pyv3"
/>
@@ -142,7 +142,7 @@ For example:
text={TSCodeLegacy}
startMarker="// START BQWithOptions"
endMarker="// END BQWithOptions"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/configuration/compression/pq-compression.md b/developers/weaviate/configuration/compression/pq-compression.md
index 7f2b37db9c..1541371d8e 100644
--- a/developers/weaviate/configuration/compression/pq-compression.md
+++ b/developers/weaviate/configuration/compression/pq-compression.md
@@ -71,7 +71,7 @@ To configure PQ in a collection, use the [PQ parameters](./pq-compression.md#pq-
text={PyCodeV3}
startMarker="# START CollectionWithAutoPQ"
endMarker="# END CollectionWithAutoPQ"
- language="py"
+ language="pyv3"
/>
@@ -89,7 +89,7 @@ To configure PQ in a collection, use the [PQ parameters](./pq-compression.md#pq-
text={TSCodeLegacy}
startMarker="// START CollectionWithAutoPQ"
endMarker="// END CollectionWithAutoPQ"
- language="ts"
+ language="tsv2"
/>
@@ -144,7 +144,7 @@ Follow these steps to manually enable PQ.
text={PyCodeV3}
startMarker="# START InitialSchema"
endMarker="# END InitialSchema"
- language="py"
+ language="pyv3"
/>
@@ -162,7 +162,7 @@ Follow these steps to manually enable PQ.
text={TSCodeLegacy}
startMarker="// START InitClassDef"
endMarker="// END InitClassDef"
- language="ts"
+ language="tsv2"
/>
@@ -218,7 +218,7 @@ To enable PQ, update your collection definition as shown below. For additional c
text={PyCodeV3}
startMarker="# START UpdateSchema"
endMarker="# END UpdateSchema"
- language="py"
+ language="pyv3"
/>
@@ -236,7 +236,7 @@ To enable PQ, update your collection definition as shown below. For additional c
text={TSCodeLegacy}
startMarker="// START UpdateSchema"
endMarker="// END UpdateSchema"
- language="ts"
+ language="tsv2"
/>
@@ -323,7 +323,7 @@ To review the current `pq` configuration, you can retrieve it as shown below.
text={PyCodeV3}
startMarker="# START GetSchema"
endMarker="# END GetSchema"
- language="py"
+ language="pyv3"
/>
@@ -341,7 +341,7 @@ To review the current `pq` configuration, you can retrieve it as shown below.
text={TSCodeLegacy}
startMarker="// START GetSchema"
endMarker="// END GetSchema"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/configuration/compression/sq-compression.md b/developers/weaviate/configuration/compression/sq-compression.md
index 5c4d4d3626..ed65251608 100644
--- a/developers/weaviate/configuration/compression/sq-compression.md
+++ b/developers/weaviate/configuration/compression/sq-compression.md
@@ -44,7 +44,7 @@ To enable SQ, set `vector_index_config`.
text={PyCodeV3}
startMarker="# START EnableSQ"
endMarker="# END EnableSQ"
- language="py"
+ language="pyv3"
/>
@@ -78,7 +78,7 @@ To tune SQ, set these `vectorIndexConfig` parameters.
text={PyCodeV3}
startMarker="# START SQWithOptions"
endMarker="# END SQWithOptions"
- language="py"
+ language="pyv3"
/>
diff --git a/developers/weaviate/configuration/modules.md b/developers/weaviate/configuration/modules.md
index 014541e85a..d50848fabc 100644
--- a/developers/weaviate/configuration/modules.md
+++ b/developers/weaviate/configuration/modules.md
@@ -18,7 +18,7 @@ At the instance (i.e. Weaviate cluster) level, you can:
This can be done by setting the appropriate [environment variables](../config-refs/env-vars.md) as shown below.
:::tip What about WCD?
-Weaviate Cloud (WCD) instances come with modules pre-configured. See [this page](../../wcs/index.mdx#configuration) for details.
+Weaviate Cloud (WCD) instances come with modules pre-configured. See [this page](../../wcs/cluster-status.mdx#enabled-modules) for details.
:::
### Enable individual modules
diff --git a/developers/weaviate/configuration/persistence.md b/developers/weaviate/configuration/persistence.md
index a18557e65d..e6df0c04a0 100644
--- a/developers/weaviate/configuration/persistence.md
+++ b/developers/weaviate/configuration/persistence.md
@@ -64,8 +64,7 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'none'
- ENABLE_MODULES: ''
+ ENABLE_API_BASED_MODULES: 'true'
CLUSTER_HOSTNAME: 'node1' # <== this can be set to an arbitrary name
...
```
diff --git a/developers/weaviate/configuration/roles.md b/developers/weaviate/configuration/roles.md
new file mode 100644
index 0000000000..57ec52e8b2
--- /dev/null
+++ b/developers/weaviate/configuration/roles.md
@@ -0,0 +1,796 @@
+---
+title: RBAC Roles
+sidebar_position: 38
+image: og/docs/configuration.jpg
+# tags: ['rbac', 'roles', 'configuration', 'authorization']
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyCode from '!!raw-loader!/_includes/code/python/howto.configure.rbac.permissions.py';
+
+:::caution RBAC technical preview
+Role-based access control (RBAC) is added `v1.28` as a **technical preview**. This means that the feature is still under development and may change in future releases, including potential breaking changes. **We do not recommend using this feature in production environments at this time.**
+
+
+We appreciate [your feedback](https://forum.weaviate.io/) on this feature.
+:::
+
+Weaviate provides differentiated access through [authorization](./authorization.md) levels, based on the [authenticated](./authentication.md) user identity.
+
+If role-based access control (RBAC) is enabled, access can be further restricted based the roles of users. In Weaviate, RBAC allows you to define roles and assign permissions to those roles. Users can then be assigned to roles, and inherit the permissions associated with those roles.
+
+Roles and permissions can be managed through Weaviate API (e.g. via REST API directly, or through a client library).
+
+Refer to the client library examples below, or [the REST API documentation](../api/rest.md) for concrete examples on how to manage roles and permissions.
+
+## Roles
+
+### Predefined roles
+
+Weaviate comes with a set of predefined roles. These roles are:
+
+- `admin`: The admin role has full access to all resources in Weaviate.
+- `viewer`: The viewer role has read-only access to all resources in Weaviate.
+
+`admin` and `viewer` roles can be assigned through the Weaviate configuration file. A predefined role cannot be modified. The user can, however, be assigned additional roles through the Weaviate API.
+
+All roles can also be assigned through the Weaviate API, including the predefined roles. The predefined roles cannot be modified, but they can be assigned or revoked from users.
+
+Refer to the [authorization](./authorization.md) page for more information on how to assign predefined roles to users.
+
+### Custom roles
+
+Any authenticated user that is not assigned a predefined role has no roles or permissions by default.
+
+These users' permissions can be modified through Weaviate by those with the appropriate permissions. This allows for the creation of custom roles, which can be assigned to users as needed.
+
+### Managing roles
+
+Role management requires appropriate `role` resource permissions through a predefined `admin` role or a role with `manage_roles` permission.
+
+## Permissions
+
+Permissions in Weaviate define what actions users can perform on specific resources. Each permission consists of:
+
+- A resource type (e.g., collections, objects)
+- Access levels (read, write, update, delete, manage)
+- Optional resource-specific constraints
+
+### Available permissions
+
+Permissions can be defined for the following resources:
+
+1. **Role Management**
+ - Read roles
+ - Manage roles
+
+1. **Collections** (collection definitions only, data object permissions are separate)
+ - Create, read, update, and delete collection definitions
+
+1. **Data Objects**
+ - Read, write, update, and delete objects
+
+1. **Backup**
+ - Manage backups
+
+1. **Cluster Data Access**
+ - Read cluster metadata
+
+1. **Node Data Access**
+ - Read node metadata at a specified verbosity level
+
+:::caution Role Management Permissions
+Be careful when assigning permissions to roles that manage roles. These permissions can be used to escalate privileges by assigning additional roles to users. Only assign these permissions to trusted users.
+:::
+
+### Permission behavior
+
+When defining permissions, setting a permission to `False` indicates that the permission is *not set*, rather than explicitly denying access. This means that if a user has multiple roles, and one role grants a permission while another sets it to `False`, the user will still have that permission through the role that grants it.
+
+For example, if a user has two roles:
+- Role A sets `read` to `False` for Collection X
+- Role B sets `read` to `True` for Collection X
+
+The user will have read access to Collection X because Role B grants the permission, while Role A's `False` value simply indicates no permission is set rather than blocking access.
+
+:::info Permissions and `aggregate` queries
+
+In `v1.28`, aggregate queries require the user to have the `read_config` permission on all collections (i.e. with a `"*"` collection name filter).
+
+
+This is due to the implementation of the `aggregate` queries under-the-hood, which uses GraphQL rather than gRPC. This will be improved in future versions of Weaviate.
+:::
+
+#### Wildcards in permissions
+
+Many permissions require a collection name filter, to specify which collections the permission applies to.
+
+In thi case, `"*"` acts as a multi-character wildcard. As an example, setting a permission with `"Test_*"` as the collection filter would apply that permission to all collections that start with `Test_`. Or, setting a permission with `"*"` as the collection filter would apply that permission to all available collections.
+
+## Example permission sets
+
+### Read and write permissions
+
+This confers read and write permissions for collections starting with `TargetCollection_`, and read permissions to nodes and cluster metadata.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Viewer permissions
+
+This confers viewer permissions for collections starting with `TargetCollection_`.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+import RolePyCode from '!!raw-loader!/_includes/code/python/howto.configure.rbac.roles.py';
+
+## Role management
+
+Role management requires appropriate `role` resource permissions through a predefined `admin` role or a role with `manage_roles` permission.
+
+For more information see the [Authentication](./authentication.md) and [Authorization](./authorization.md) pages.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+
+
+### Create new role with permissions
+
+This example creates a role called "devrel" with permissions to:
+
+- Read all collections starting with "Test_".
+- Delete or create the collection "Test_DevRel"
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Grant additional permissions
+
+Additional permissions can be granted to a role at any time. The role must already exist.
+
+This example grants additional permissions to the "devrel" role to:
+
+- Read data in all collections that start with "Test_".
+- Create new data in "Test_DevRel".
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Remove permissions from a role
+
+Permissions can be removed from a role at any time. Removing all permissions from a role will not delete the role itself.
+
+This example removes from the "devrel" role permissions to:
+
+- Read the data from collections that start with "Test_"
+- Create and delete collections called "Test_DevRel"
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Assign a role to a user
+
+A custom user can have any number of roles assigned to them (including none). The role can be a predefined role (e.g. `viewer`) or a custom role.
+
+This example assigns the custom "devrel" role to "jane-doe".
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Get the current user's roles and permissions
+
+Retrieve the role and permission information for the currently authenticated user.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Get a user's roles and permissions
+
+Retrieve the role and permission information for any user.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Check if a role exists
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Inspect a role
+
+View the permissions assigned to a role.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### List all roles
+
+View all roles in the system and their permissions.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### List users with a role
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Delete a role
+
+Deleting a role will remove it from the system, and revoke the associated permissions from all users had this role.
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+### Revoke a role from a user
+
+You can revoke one or more roles from a specific user.
+
+This examples revokes "role-1" and "role-2" from the user "jane-doe".
+
+
+
+
+
+
+
+
+
+```ts
+// TS support coming soon
+```
+
+
+
+
+
+```go
+// Go support coming soon
+```
+
+
+
+
+
+```java
+// Java support coming soon
+```
+
+
+
+
+
+## Further resources
+
+- [Configuration: Authentication](./authentication.md)
+- [Configuration: Authorization and RBAC](./authorization.md)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/connections/connect-cloud.mdx b/developers/weaviate/connections/connect-cloud.mdx
index f48e6886bd..05f3dc18d5 100644
--- a/developers/weaviate/connections/connect-cloud.mdx
+++ b/developers/weaviate/connections/connect-cloud.mdx
@@ -45,7 +45,7 @@ To connect, use the Weaviate URL and the Weaviate API key for your WCD instance
text={PyCodeV3}
startMarker="# START APIKeyWCD"
endMarker="# END APIKeyWCD"
- language="py"
+ language="pyv3"
/>
@@ -61,7 +61,7 @@ To connect, use the Weaviate URL and the Weaviate API key for your WCD instance
text={TsCodeV2}
startMarker="// START APIKeyWCD"
endMarker="// END APIKeyWCD"
- language="js"
+ language="tsv2"
/>
@@ -108,7 +108,7 @@ If you use API-based models for vectorization or RAG, you must provide an API ke
text={PyCodeV3}
startMarker="# START ThirdPartyAPIKeys"
endMarker="# END ThirdPartyAPIKeys"
- language="py"
+ language="pyv3"
/>
@@ -124,7 +124,7 @@ If you use API-based models for vectorization or RAG, you must provide an API ke
text={TsCodeV2}
startMarker="// START ThirdPartyAPIKeys"
endMarker="// END ThirdPartyAPIKeys"
- language="js"
+ language="tsv2"
/>
diff --git a/developers/weaviate/connections/connect-embedded.mdx b/developers/weaviate/connections/connect-embedded.mdx
index f96d6c2c50..44c0db39f5 100644
--- a/developers/weaviate/connections/connect-embedded.mdx
+++ b/developers/weaviate/connections/connect-embedded.mdx
@@ -35,7 +35,7 @@ For more details on using Embedded Weaviate, see [Embedded Weaviate](/developers
text={PyCodeV3}
startMarker="# START Embedded"
endMarker="# END Embedded"
- language="py"
+ language="pyv3"
/>
@@ -43,7 +43,7 @@ For more details on using Embedded Weaviate, see [Embedded Weaviate](/developers
text={TsCodeV2}
startMarker="// START Embedded"
endMarker="// END Embedded"
- language="js"
+ language="tsv2"
/>
diff --git a/developers/weaviate/connections/connect-local.mdx b/developers/weaviate/connections/connect-local.mdx
index 961aa3e91b..f42a8ad80e 100644
--- a/developers/weaviate/connections/connect-local.mdx
+++ b/developers/weaviate/connections/connect-local.mdx
@@ -47,7 +47,7 @@ To connect to a local instance without authentication, follow these examples.
text={PyCodeV3}
startMarker="# START LocalNoAuth"
endMarker="# END LocalNoAuth"
- language="py"
+ language="pyv3"
/>
@@ -63,7 +63,7 @@ To connect to a local instance without authentication, follow these examples.
text={TsCodeV2}
startMarker="// START LocalNoAuth"
endMarker="// END LocalNoAuth"
- language="js"
+ language="tsv2"
/>
@@ -110,7 +110,7 @@ To change the default URL or port number, follow these examples.
text={PyCodeV3}
startMarker="# START CustomURL"
endMarker="# END CustomURL"
- language="py"
+ language="pyv3"
/>
@@ -126,7 +126,7 @@ To change the default URL or port number, follow these examples.
text={TsCodeV2}
startMarker="// START CustomURL"
endMarker="// END CustomURL"
- language="js"
+ language="tsv2"
/>
@@ -174,7 +174,7 @@ To authenticate with a Weaviate API key, follow these examples.
text={PyCodeV3}
startMarker="# START LocalAuth"
endMarker="# END LocalAuth"
- language="py"
+ language="pyv3"
/>
@@ -190,7 +190,7 @@ To authenticate with a Weaviate API key, follow these examples.
text={TsCodeV2}
startMarker="// START LocalAuth"
endMarker="// END LocalAuth"
- language="js"
+ language="tsv2"
/>
@@ -245,7 +245,7 @@ Integrations that use external APIs often need API keys. To add third party API
text={PyCodeV3}
startMarker="# START LocalThirdPartyAPIKeys"
endMarker="# END LocalThirdPartyAPIKeys"
- language="py"
+ language="pyv3"
/>
@@ -261,7 +261,7 @@ Integrations that use external APIs often need API keys. To add third party API
text={TsCodeV2}
startMarker="// START LocalThirdPartyAPIKeys"
endMarker="// END LocalThirdPartyAPIKeys"
- language="js"
+ language="tsv2"
/>
diff --git a/developers/weaviate/installation/aws-marketplace.md b/developers/weaviate/installation/aws-marketplace.md
index 0f6775858f..1726f0324f 100644
--- a/developers/weaviate/installation/aws-marketplace.md
+++ b/developers/weaviate/installation/aws-marketplace.md
@@ -1,5 +1,6 @@
---
title: AWS Marketplace
+description: Install Weaviate through the AWS Marketplace for quick cloud deployment.
sidebar_position: 10
image: og/docs/installation.jpg
# tags: ['installation', 'AWS Marketplace']
@@ -7,8 +8,6 @@ image: og/docs/installation.jpg
import ReactPlayer from 'react-player/lazy'
-## Overview
-
You can use [AWS Marketplace](https://aws.amazon.com/marketplace) to directly launch a Weaviate cluster.
diff --git a/developers/weaviate/installation/docker-compose.md b/developers/weaviate/installation/docker-compose.md
index 6d36584306..69af6d73a6 100644
--- a/developers/weaviate/installation/docker-compose.md
+++ b/developers/weaviate/installation/docker-compose.md
@@ -65,7 +65,6 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'none'
ENABLE_API_BASED_MODULES: 'true'
CLUSTER_HOSTNAME: 'node1'
volumes:
@@ -150,7 +149,6 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'none'
CLUSTER_HOSTNAME: 'node1'
```
@@ -266,8 +264,7 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- ENABLE_MODULES: 'text2vec-openai,text2vec-cohere,text2vec-huggingface'
- DEFAULT_VECTORIZER_MODULE: 'none'
+ ENABLE_API_BASED_MODULES: 'true'
CLUSTER_HOSTNAME: 'node1'
CLUSTER_GOSSIP_BIND_PORT: '7100'
CLUSTER_DATA_BIND_PORT: '7101'
@@ -296,8 +293,7 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- ENABLE_MODULES: 'text2vec-openai,text2vec-cohere,text2vec-huggingface'
- DEFAULT_VECTORIZER_MODULE: 'none'
+ ENABLE_API_BASED_MODULES: 'true'
CLUSTER_HOSTNAME: 'node2'
CLUSTER_GOSSIP_BIND_PORT: '7102'
CLUSTER_DATA_BIND_PORT: '7103'
@@ -327,8 +323,7 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- ENABLE_MODULES: 'text2vec-openai,text2vec-cohere,text2vec-huggingface'
- DEFAULT_VECTORIZER_MODULE: 'none'
+ ENABLE_API_BASED_MODULES: 'true'
CLUSTER_HOSTNAME: 'node3'
CLUSTER_GOSSIP_BIND_PORT: '7104'
CLUSTER_DATA_BIND_PORT: '7105'
diff --git a/developers/weaviate/installation/gc-marketplace.md b/developers/weaviate/installation/gc-marketplace.md
index 4c5584d9a2..6330734aa2 100644
--- a/developers/weaviate/installation/gc-marketplace.md
+++ b/developers/weaviate/installation/gc-marketplace.md
@@ -1,13 +1,11 @@
---
title: Google Cloud Marketplace
+description: Set up Weaviate using Google Cloud Marketplace for simplified deployment.
sidebar_position: 15
image: og/docs/installation.jpg
tags: ['installation', 'Google Cloud Marketplace']
---
-
-## Overview
-
You can use [Google Cloud Marketplace](https://console.cloud.google.com/marketplace) to directly launch a Weaviate cluster.
:::info Prerequisites
diff --git a/developers/weaviate/installation/index.md b/developers/weaviate/installation/index.md
index 6f2ef90528..a532255a6f 100644
--- a/developers/weaviate/installation/index.md
+++ b/developers/weaviate/installation/index.md
@@ -7,6 +7,8 @@ image: og/docs/installation.jpg
Weaviate is available as a hosted service, [Weaviate Cloud (WCD)](https://console.weaviate.cloud/), or as a self managed instance. If you manage your own instance, you can host it locally or with a cloud provider. Self-managed instances use the same Weaviate core database as WCD.
+If you are upgrading from a previous version of Weaviate, see the [Migration Guide](../more-resources/migration/index.md) for any changes that may affect your installation.
+
## Installation methods
To install and configure Weaviate, see the following:
@@ -35,7 +37,7 @@ import RunUnreleasedImages from '/_includes/configuration/run-unreleased.mdx'
When you try upcoming features, please provide [feedback](https://github.com/weaviate/weaviate/issues/new/choose). Your comments are appreciated and help us to make Weaviate more useful for you.
## Related pages
-- [Connect to Weaviate](../connections)
+- [Connect to Weaviate](../connections/index.mdx)
- [Weaviate Quickstart](../quickstart/index.md)
- [Weaviate Cloud Quickstart](../../wcs/quickstart.mdx)
- [References: Configuration](../configuration/index.md)
diff --git a/developers/weaviate/installation/weaviate-cloud-services.md b/developers/weaviate/installation/weaviate-cloud-services.md
index 1011cb6742..da20b7ea0a 100644
--- a/developers/weaviate/installation/weaviate-cloud-services.md
+++ b/developers/weaviate/installation/weaviate-cloud-services.md
@@ -1,5 +1,6 @@
---
title: Weaviate Cloud
+description: Installing Weaviate Cloud Services
sidebar_position: 1
image: og/docs/installation.jpg
# tags: ['installation', 'Weaviate Cloud']
diff --git a/developers/weaviate/introduction.md b/developers/weaviate/introduction.md
index 5935218f42..1354b5a112 100644
--- a/developers/weaviate/introduction.md
+++ b/developers/weaviate/introduction.md
@@ -76,7 +76,7 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.class
text={PyCodeV3}
startMarker="# START ReadAllCollections"
endMarker="# END ReadAllCollections"
- language="py"
+ language="pyv3"
/>
@@ -94,7 +94,7 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.class
text={TSCodeLegacy}
startMarker="// START ReadAllCollections"
endMarker="// END ReadAllCollections"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/manage-data/collections.mdx b/developers/weaviate/manage-data/collections.mdx
index 8f0c9ae78f..631fc52466 100644
--- a/developers/weaviate/manage-data/collections.mdx
+++ b/developers/weaviate/manage-data/collections.mdx
@@ -44,7 +44,7 @@ import InitialCaps from '/_includes/schemas/initial-capitalization.md'
text={PyCodeV3}
startMarker="# START BasicCreateCollection"
endMarker="# END BasicCreateCollection"
- language="py"
+ language="pyv3"
/>
@@ -62,7 +62,7 @@ import InitialCaps from '/_includes/schemas/initial-capitalization.md'
text={TSCodeLegacy}
startMarker="// START BasicCreateCollection"
endMarker="// END BasicCreateCollection"
- language="ts"
+ language="tsv2"
/>
@@ -119,7 +119,7 @@ For details, see:
text={PyCodeV3}
startMarker="# START CreateCollectionWithProperties"
endMarker="# END CreateCollectionWithProperties"
- language="py"
+ language="pyv3"
/>
@@ -137,7 +137,7 @@ For details, see:
text={TSCodeLegacy}
startMarker="// START CreateCollectionWithProperties"
endMarker="// END CreateCollectionWithProperties"
- language="ts"
+ language="tsv2"
/>
@@ -187,7 +187,7 @@ Collection level settings override default values and general configuration para
text={PyCodeV3}
startMarker="# START Vectorizer"
endMarker="# END Vectorizer"
- language="py"
+ language="pyv3"
/>
@@ -205,7 +205,7 @@ Collection level settings override default values and general configuration para
text={TSCodeLegacy}
startMarker="// START Vectorizer"
endMarker="// END Vectorizer"
- language="ts"
+ language="tsv2"
/>
@@ -243,7 +243,7 @@ You can define multiple named vectors per collection. This allows each object to
text={PyCodeV3}
startMarker="# START BasicNamedVectors"
endMarker="# END BasicNamedVectors"
- language="py"
+ language="pyv3"
/>
@@ -261,7 +261,7 @@ You can define multiple named vectors per collection. This allows each object to
text={TSCodeLegacy}
startMarker="// START BasicNamedVectors"
endMarker="// END BasicNamedVectors"
- language="ts"
+ language="tsv2"
/>
@@ -295,7 +295,7 @@ To configure how a vectorizer works (i.e. what model to use) with a specific col
text={PyCodeV3}
startMarker="# START ModuleSettings"
endMarker="# END ModuleSettings"
- language="py"
+ language="pyv3"
/>
@@ -313,7 +313,7 @@ To configure how a vectorizer works (i.e. what model to use) with a specific col
text={TSCodeLegacy}
startMarker="// START ModuleSettings"
endMarker="// END ModuleSettings"
- language="ts"
+ language="tsv2"
/>
@@ -347,7 +347,7 @@ The vector index type can be set for each collection at creation time, between `
text={PyCodeV3}
startMarker="# START SetVectorIndexType"
endMarker="# END SetVectorIndexType"
- language="py"
+ language="pyv3"
/>
@@ -365,7 +365,7 @@ The vector index type can be set for each collection at creation time, between `
text={TSCodeLegacy}
startMarker="// START SetVectorIndexType"
endMarker="// END SetVectorIndexType"
- language="ts"
+ language="tsv2"
/>
@@ -409,7 +409,7 @@ Was added in `v1.27`
text={PyCodeV3}
startMarker="# START SetVectorIndexParams"
endMarker="# END SetVectorIndexParams"
- language="py"
+ language="pyv3"
/>
@@ -427,7 +427,7 @@ Was added in `v1.27`
text={TSCodeLegacy}
startMarker="// START SetVectorIndexParams"
endMarker="// END SetVectorIndexParams"
- language="ts"
+ language="tsv2"
/>
@@ -471,7 +471,7 @@ Configure individual properties in a collection. Each property can have it's own
text={PyCodeV3}
startMarker="# START PropModuleSettings"
endMarker="# END PropModuleSettings"
- language="py"
+ language="pyv3"
/>
@@ -489,7 +489,7 @@ Configure individual properties in a collection. Each property can have it's own
text={TSCodeLegacy}
startMarker="// START PropModuleSettings"
endMarker="// END PropModuleSettings"
- language="ts"
+ language="tsv2"
/>
@@ -522,7 +522,7 @@ If you choose to bring your own vectors, you should specify the `distance metric
text={PyCodeV3}
startMarker="# START DistanceMetric"
endMarker="# END DistanceMetric"
- language="py"
+ language="pyv3"
/>
@@ -540,7 +540,7 @@ If you choose to bring your own vectors, you should specify the `distance metric
text={TSCodeLegacy}
startMarker="// START DistanceMetric"
endMarker="// END DistanceMetric"
- language="ts"
+ language="tsv2"
/>
@@ -585,7 +585,7 @@ Various [inverted index parameters](../config-refs/schema/index.md#invertedindex
text={PyCodeV3}
startMarker="# START SetInvertedIndexParams"
endMarker="# END SetInvertedIndexParams"
- language="py"
+ language="pyv3"
/>
@@ -603,14 +603,18 @@ Various [inverted index parameters](../config-refs/schema/index.md#invertedindex
text={TSCodeLegacy}
startMarker="// START SetInvertedIndexParams"
endMarker="// END SetInvertedIndexParams"
- language="ts"
+ language="tsv2"
/>
-## Specify a reranker
+## Specify a reranker model integration
+
+Configure a [`reranker`](../concepts/search/index.md#rerank) model integration for [reranking retrieved results](../search/rerank.md).
-Configure a [`reranker`](../concepts/search/index.md#rerank) for [reranking retrieved results](../search/rerank.md).
+:::info Related pages
+- [Available reranker model integrations](../model-providers/index.md)
+:::
@@ -627,7 +631,7 @@ Configure a [`reranker`](../concepts/search/index.md#rerank) for [reranking retr
text={PyCodeV3}
startMarker="# START SetReranker"
endMarker="# END SetReranker"
- language="py"
+ language="pyv3"
/>
@@ -645,21 +649,63 @@ Configure a [`reranker`](../concepts/search/index.md#rerank) for [reranking retr
text={TSCodeLegacy}
startMarker="// START SetReranker"
endMarker="// END SetReranker"
- language="ts"
+ language="tsv2"
/>
-## Specify a generative module
+## Update the reranker model integration
-Specify a `generative` module for a collection (for RAG).
+:::info Available from `v1.25.23`, `v1.26.8` and `v1.27.1`
+The `reranker` and `generative` configurations are mutable from `v1.25.23`, `v1.26.8` and `v1.27.1`.
+:::
-
- Additional information
+Update the [`reranker`](../concepts/search/index.md#rerank) model integration for [reranking retrieved results](../search/rerank.md).
-- [Available generative model integrations](../model-providers/index.md)
+
+
+
+
-
+
+
+
+
+
+
+```js
+// Coming soon
+```
+
+
+
+
+
+
+
+
+## Specify a generative model integration
+
+Specify a `generative` model integration for a collection (for RAG).
+
+:::info Related pages
+- [Available generative model integrations](../model-providers/index.md)
+:::
@@ -676,7 +722,7 @@ Specify a `generative` module for a collection (for RAG).
text={PyCodeV3}
startMarker="# START SetGenerative"
endMarker="# END SetGenerative"
- language="py"
+ language="pyv3"
/>
@@ -694,7 +740,7 @@ Specify a `generative` module for a collection (for RAG).
text={TSCodeLegacy}
startMarker="// START SetGenerative"
endMarker="// END SetGenerative"
- language="ts"
+ language="tsv2"
/>
@@ -718,7 +764,7 @@ Specify a generative `model name`.
text={PyCodeV3}
startMarker="# START SetGenModel"
endMarker="# END SetGenModel"
- language="py"
+ language="pyv3"
/>
@@ -736,7 +782,52 @@ Specify a generative `model name`.
text={TSCodeLegacy}
startMarker="// START SetGenModel"
endMarker="// END SetGenModel"
- language="ts"
+ language="tsv2"
+ />
+
+
+
+## Update the generative model integration
+
+:::info Available from `v1.25.23`, `v1.26.8` and `v1.27.1`
+The `reranker` and `generative` configurations are mutable from `v1.25.23`, `v1.26.8` and `v1.27.1`.
+:::
+
+Update a [`reranker`](../concepts/search/index.md#rerank) model integration for [reranking retrieved results](../search/rerank.md).
+
+
+
+
+
+
+
+
+
+
+
+
+```js
+// Coming soon
+```
+
+
+
+
+
@@ -747,11 +838,74 @@ import RaftRFChangeWarning from '/_includes/1-25-replication-factor.mdx';
-Configure replication per collection.
+Configure replication settings, such as [async replication](../concepts/replication-architecture/consistency.md#async-replication) and [deletion resolution strategy](../concepts/replication-architecture/consistency.md#deletion-resolution-strategies).
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-import ReplicationConfigWithAsyncRepair from '/_includes/code/configuration/replication-consistency.mdx';
+```bash
+curl \
+-X POST \
+-H "Content-Type: application/json" \
+-d '{
+ "class": "Article",
+ "properties": [
+ {
+ "dataType": [
+ "string"
+ ],
+ "description": "Title of the article",
+ "name": "title"
+ }
+ ],
+ "replicationConfig": {
+ "factor": 3,
+ "asyncEnabled": true
+ }
+}' \
+http://localhost:8080/v1/schema
+```
-
+
+
@@ -785,7 +939,7 @@ Configure sharding per collection.
text={PyCodeV3}
startMarker="# START ShardingSettings"
endMarker="# END ShardingSettings"
- language="py"
+ language="pyv3"
/>
@@ -803,7 +957,7 @@ Configure sharding per collection.
text={TSCodeLegacy}
startMarker="// START ShardingSettings"
endMarker="// END ShardingSettings"
- language="ts"
+ language="tsv2"
/>
@@ -841,7 +995,7 @@ Create a collection with multi-tenancy enabled.
text={PyCodeV3}
startMarker="# START Multi-tenancy"
endMarker="# END Multi-tenancy"
- language="py"
+ language="pyv3"
/>
@@ -859,7 +1013,7 @@ Create a collection with multi-tenancy enabled.
text={TSCodeLegacy}
startMarker="// START Multi-tenancy"
endMarker="// END Multi-tenancy"
- language="ts"
+ language="tsv2"
/>
@@ -882,7 +1036,7 @@ Retrieve a collection definition from the schema.
text={PyCodeV3}
startMarker="# START ReadOneCollection"
endMarker="# END ReadOneCollection"
- language="py"
+ language="pyv3"
/>
@@ -900,7 +1054,7 @@ Retrieve a collection definition from the schema.
text={TSCodeLegacy}
startMarker="// START ReadOneCollection"
endMarker="// END ReadOneCollection"
- language="ts"
+ language="tsv2"
/>
@@ -1107,7 +1261,7 @@ Fetch the database schema to retrieve all of the collection definitions.
text={PyCodeV3}
startMarker="# START ReadAllCollections"
endMarker="# END ReadAllCollections"
- language="py"
+ language="pyv3"
/>
@@ -1125,7 +1279,7 @@ Fetch the database schema to retrieve all of the collection definitions.
text={TSCodeLegacy}
startMarker="// START ReadAllCollections"
endMarker="// END ReadAllCollections"
- language="ts"
+ language="tsv2"
/>
@@ -1169,7 +1323,7 @@ You can update a collection definition to change the [mutable collection setting
text={PyCodeV3}
startMarker="# START UpdateCollection"
endMarker="# END UpdateCollection"
- language="py"
+ language="pyv3"
/>
@@ -1183,11 +1337,12 @@ You can update a collection definition to change the [mutable collection setting
-
-```ts
-// This feature is under development
-```
-
+
@@ -1226,7 +1381,7 @@ Some parameters cannot be modified after you create your collection.
text={PyCodeV3}
startMarker="# START ModifyParam"
endMarker="# END ModifyParam"
- language="py"
+ language="pyv3"
/>
diff --git a/developers/weaviate/manage-data/create.mdx b/developers/weaviate/manage-data/create.mdx
index 418c04300e..2092e2f601 100644
--- a/developers/weaviate/manage-data/create.mdx
+++ b/developers/weaviate/manage-data/create.mdx
@@ -18,6 +18,10 @@ import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.creat
The examples on this page demonstrate how to create individual objects in Weaviate.
+:::tip Use batch import for multiple objects
+For creating multiple objects at once, see [How-to: Batch Import](./import.mdx).
+:::
+
## Create an object
This example creates an object in the `JeopardyQuestion` collection.
@@ -37,7 +41,7 @@ This example creates an object in the `JeopardyQuestion` collection.
text={PyCodeV3}
startMarker="# CreateObject START"
endMarker="# CreateObject END"
- language="py"
+ language="pyv3"
/>
@@ -55,7 +59,7 @@ This example creates an object in the `JeopardyQuestion` collection.
text={TSCodeLegacy}
startMarker="// CreateObject START"
endMarker="// CreateObject END"
- language="ts"
+ language="tsv2"
/>
@@ -112,7 +116,7 @@ When you create an object, you can provide a vector. (For specifying multiple, n
text={PyCodeV3}
startMarker="# CreateObjectWithVector START"
endMarker="# CreateObjectWithVector END"
- language="py"
+ language="pyv3"
/>
@@ -130,7 +134,7 @@ When you create an object, you can provide a vector. (For specifying multiple, n
text={TSCodeLegacy}
startMarker="// CreateObjectWithVector START"
endMarker="// CreateObjectWithVector END"
- language="ts"
+ language="tsv2"
/>
@@ -175,7 +179,7 @@ When you create an object, you can specify named vectors (if [configured in your
text={PyCodeV3}
startMarker="# CreateObjectNamedVectors START"
endMarker="# CreateObjectNamedVectors END"
- language="py"
+ language="pyv3"
/>
@@ -193,7 +197,7 @@ When you create an object, you can specify named vectors (if [configured in your
text={TSCodeLegacy}
startMarker="// CreateObjectNamedVectors START"
endMarker="// CreateObjectNamedVectors END"
- language="ts"
+ language="tsv2"
/>
@@ -221,7 +225,7 @@ If no ID is provided, Weaviate will generate a random [UUID](https://en.wikipedi
text={PyCodeV3}
startMarker="# CreateObjectWithId START"
endMarker="# CreateObjectWithId END"
- language="py"
+ language="pyv3"
/>
@@ -239,7 +243,7 @@ If no ID is provided, Weaviate will generate a random [UUID](https://en.wikipedi
text={TSCodeLegacy}
startMarker="// CreateObjectWithId START"
endMarker="// CreateObjectWithId END"
- language="ts"
+ language="tsv2"
/>
@@ -285,7 +289,7 @@ Object IDs are not randomly generated. The same value always generates the same
text={PyCodeV3}
startMarker="# CreateObjectWithDeterministicId START"
endMarker="# CreateObjectWithDeterministicId END"
- language="py"
+ language="pyv3"
/>
@@ -303,7 +307,7 @@ Object IDs are not randomly generated. The same value always generates the same
text={TSCodeLegacy}
startMarker="// CreateObjectWithDeterministicId START"
endMarker="// CreateObjectWithDeterministicId END"
- language="ts"
+ language="tsv2"
/>
@@ -360,7 +364,7 @@ You can create an object with cross-references to other objects.
text={XrefPyCodeV3}
startMarker="# START ObjectWithCrossRef"
endMarker="# END ObjectWithCrossRef"
- language="py"
+ language="pyv3"
/>
@@ -378,7 +382,7 @@ You can create an object with cross-references to other objects.
text={XrefTSCodeLegacy}
startMarker="// START ObjectWithCrossRef"
endMarker="// END ObjectWithCrossRef"
- language="ts"
+ language="tsv2"
/>
@@ -418,7 +422,7 @@ Before you create an object, you can [validate](/developers/weaviate/api/rest#ta
text={PyCodeV3}
startMarker="# ValidateObject START"
endMarker="# ValidateObject END"
- language="py"
+ language="pyv3"
/>
@@ -436,7 +440,7 @@ Before you create an object, you can [validate](/developers/weaviate/api/rest#ta
text={TSCodeLegacy}
startMarker="// ValidateObject START"
endMarker="// ValidateObject END"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/manage-data/cross-references.mdx b/developers/weaviate/manage-data/cross-references.mdx
index 591cd45066..11937b7e80 100644
--- a/developers/weaviate/manage-data/cross-references.mdx
+++ b/developers/weaviate/manage-data/cross-references.mdx
@@ -51,7 +51,7 @@ Include the reference property in the collection definition before adding cross-
text={PyCodeV3}
startMarker="# START CrossRefDefinition"
endMarker="# END CrossRefDefinition"
- language="py"
+ language="pyv3"
/>
@@ -82,7 +82,7 @@ It is also possible to add a cross-reference property to an existing collection
text={PyCodeV3}
startMarker="# START Collections TwoWay Category2"
endMarker="# END Collections TwoWay Category2"
- language="python"
+ language="pyv3"
/>
@@ -99,7 +99,7 @@ It is also possible to add a cross-reference property to an existing collection
text={TSCodeLegacy}
startMarker="// START Collections TwoWay Category2"
endMarker="// END Collections TwoWay Category2"
- language="ts"
+ language="tsv2"
/>
@@ -124,7 +124,7 @@ Specify a cross-reference when creating an object.
text={PyCodeV3}
startMarker="# START ObjectWithCrossRef"
endMarker="# END ObjectWithCrossRef"
- language="py"
+ language="pyv3"
/>
@@ -142,7 +142,7 @@ Specify a cross-reference when creating an object.
text={TSCodeLegacy}
startMarker="// START ObjectWithCrossRef"
endMarker="// END ObjectWithCrossRef"
- language="ts"
+ language="tsv2"
/>
@@ -167,7 +167,7 @@ Specify the required id and properties for the source and the target.
text={PyCodeV3}
startMarker="# START OneWay"
endMarker="# END OneWay"
- language="py"
+ language="pyv3"
/>
@@ -185,7 +185,7 @@ Specify the required id and properties for the source and the target.
text={TSCodeLegacy}
startMarker="// START OneWay"
endMarker="// END OneWay"
- language="ts"
+ language="tsv2"
/>
@@ -229,7 +229,7 @@ Create the `JeopardyCategory` collection:
text={PyCodeV3}
startMarker="# START Collections TwoWay Category1"
endMarker="# END Collections TwoWay Category1"
- language="python"
+ language="pyv3"
/>
@@ -246,7 +246,7 @@ Create the `JeopardyCategory` collection:
text={TSCodeLegacy}
startMarker="// START Collections TwoWay Category1"
endMarker="// END Collections TwoWay Category1"
- language="ts"
+ language="tsv2"
/>
@@ -267,7 +267,7 @@ Create the `JeopardyQuestion` collection including the reference property to `Je
text={PyCodeV3}
startMarker="# START Collections TwoWay Question"
endMarker="# END Collections TwoWay Question"
- language="python"
+ language="pyv3"
/>
@@ -284,7 +284,7 @@ Create the `JeopardyQuestion` collection including the reference property to `Je
text={TSCodeLegacy}
startMarker="// START Collections TwoWay Question"
endMarker="// END Collections TwoWay Question"
- language="ts"
+ language="tsv2"
/>
@@ -305,7 +305,7 @@ Modify `JeopardyCategory` to add the reference to `JeopardyQuestion`:
text={PyCodeV3}
startMarker="# START Collections TwoWay Category2"
endMarker="# END Collections TwoWay Category2"
- language="python"
+ language="pyv3"
/>
@@ -322,7 +322,7 @@ Modify `JeopardyCategory` to add the reference to `JeopardyQuestion`:
text={TSCodeLegacy}
startMarker="// START Collections TwoWay Category2"
endMarker="// END Collections TwoWay Category2"
- language="ts"
+ language="tsv2"
/>
@@ -344,7 +344,7 @@ And add the cross-references:
text={PyCodeV3}
startMarker="# TwoWay Python"
endMarker="# END TwoWay Python"
- language="py"
+ language="pyv3"
/>
@@ -362,7 +362,7 @@ And add the cross-references:
text={TSCodeLegacy}
startMarker="// TwoWay TS"
endMarker="// END TwoWay TS"
- language="ts"
+ language="tsv2"
/>
@@ -405,7 +405,7 @@ Weaviate allows creation of multiple cross-references from one source object.
text={PyCodeV3}
startMarker="# Multiple Python"
endMarker="# END Multiple Python"
- language="py"
+ language="pyv3"
/>
@@ -423,7 +423,7 @@ Weaviate allows creation of multiple cross-references from one source object.
text={TSCodeLegacy}
startMarker="// Multiple TS"
endMarker="// END Multiple TS"
- language="ts"
+ language="tsv2"
/>
@@ -465,7 +465,7 @@ Cross-references can be read as part of the object.
text={PyCodeV3}
startMarker="# ReadCrossRef"
endMarker="# END ReadCrossRef"
- language="py"
+ language="pyv3"
/>
@@ -498,7 +498,7 @@ Deleting a cross-reference with the same parameters used to define the cross-ref
text={PyCodeV3}
startMarker="# Delete Python"
endMarker="# END Delete Python"
- language="py"
+ language="pyv3"
/>
@@ -516,7 +516,7 @@ Deleting a cross-reference with the same parameters used to define the cross-ref
text={TSCodeLegacy}
startMarker="// Delete TS"
endMarker="// END Delete TS"
- language="ts"
+ language="tsv2"
/>
@@ -569,7 +569,7 @@ The targets of a cross-reference can be updated.
text={PyCodeV3}
startMarker="# Update Python"
endMarker="# END Update Python"
- language="py"
+ language="pyv3"
/>
@@ -587,7 +587,7 @@ The targets of a cross-reference can be updated.
text={TSCodeLegacy}
startMarker="// Update TS"
endMarker="// END Update TS"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/manage-data/delete.mdx b/developers/weaviate/manage-data/delete.mdx
index 018657b6cb..adeae79c34 100644
--- a/developers/weaviate/manage-data/delete.mdx
+++ b/developers/weaviate/manage-data/delete.mdx
@@ -53,7 +53,7 @@ To delete by id, specify the collection name and the object id.
text={PyCodeV3}
startMarker="# START DeleteObject"
endMarker="# END DeleteObject"
- language="py"
+ language="pyv3"
/>
@@ -71,7 +71,7 @@ To delete by id, specify the collection name and the object id.
text={TSCodeLegacy}
startMarker="// START DeleteObject"
endMarker="// END DeleteObject"
- language="ts"
+ language="tsv2"
/>
@@ -116,7 +116,7 @@ You can handle errors as follows:
text={TSCode}
startMarker="// START DeleteError"
endMarker="// END DeleteError"
- language="ts"
+ language="tsv2"
/>
-->
@@ -141,7 +141,7 @@ To delete objects that match a set of criteria, specify the collection and a [`w
text={PyCodeV3}
startMarker="# START DeleteBatch"
endMarker="# END DeleteBatch"
- language="py"
+ language="pyv3"
/>
@@ -159,7 +159,7 @@ To delete objects that match a set of criteria, specify the collection and a [`w
text={TSCodeLegacy}
startMarker="// START DeleteBatch"
endMarker="// END DeleteBatch"
- language="ts"
+ language="tsv2"
/>
@@ -211,7 +211,7 @@ Use `ContainsAny` / `ContainsAll` filters to delete of objects by a set of crite
text={PyCodeV3}
startMarker="# START DeleteContains"
endMarker="# END DeleteContains"
- language="py"
+ language="pyv3"
/>
@@ -315,7 +315,7 @@ The default `QUERY_MAXIMUM_RESULTS` value is 10,000. This may be configurable, e
text={PyCodeV3}
startMarker="# START DeleteByIDBatch"
endMarker="# END DeleteByIDBatch"
- language="py"
+ language="pyv3"
/>
@@ -405,7 +405,7 @@ Objects must belong to a collection in Weaviate. Accordingly,
@@ -423,7 +423,7 @@ Objects must belong to a collection in Weaviate. Accordingly,
diff --git a/developers/weaviate/manage-data/import.mdx b/developers/weaviate/manage-data/import.mdx
index 0cf2d39962..e8d4ee5c17 100644
--- a/developers/weaviate/manage-data/import.mdx
+++ b/developers/weaviate/manage-data/import.mdx
@@ -49,7 +49,7 @@ The following example adds objects to the `MyCollection` collection.
text={PyCodeV3}
startMarker="# START BasicBatchImportExample"
endMarker="# END BasicBatchImportExample"
- language="py"
+ language="pyv3"
/>
@@ -67,7 +67,7 @@ The following example adds objects to the `MyCollection` collection.
text={TSCodeLegacy}
startMarker="// START BasicBatchImportExample"
endMarker="// END BasicBatchImportExample"
- language="ts"
+ language="tsv2"
/>
@@ -187,7 +187,7 @@ Weaviate generates an UUID for each object. Object IDs must be unique. If you se
text={PyCodeV3}
startMarker="# START BatchImportWithIDExample"
endMarker="# END BatchImportWithIDExample"
- language="py"
+ language="pyv3"
/>
@@ -205,7 +205,7 @@ Weaviate generates an UUID for each object. Object IDs must be unique. If you se
text={TSCodeLegacy}
startMarker="// START BatchImportWithIDExample"
endMarker="// END BatchImportWithIDExample"
- language="ts"
+ language="tsv2"
/>
@@ -247,7 +247,7 @@ Use the `vector` property to specify a vector for each object.
text={PyCodeV3}
startMarker="# START BatchImportWithVectorExample"
endMarker="# END BatchImportWithVectorExample"
- language="py"
+ language="pyv3"
/>
@@ -265,7 +265,7 @@ Use the `vector` property to specify a vector for each object.
text={TSCodeLegacy}
startMarker="// START BatchImportWithVectorExample"
endMarker="// END BatchImportWithVectorExample"
- language="ts"
+ language="tsv2"
/>
@@ -310,7 +310,7 @@ When you create an object, you can specify named vectors (if [configured in your
text={PyCodeV3}
startMarker="# START BatchImportWithNamedVectors"
endMarker="# END BatchImportWithNamedVectors"
- language="py"
+ language="pyv3"
/>
@@ -328,7 +328,7 @@ When you create an object, you can specify named vectors (if [configured in your
text={TSCodeLegacy}
startMarker="// START BatchImportWithNamedVectors"
endMarker="// END BatchImportWithNamedVectors"
- language="ts"
+ language="tsv2"
/>
@@ -352,7 +352,7 @@ You can batch create links from an object to another other object through cross-
text={PyCodeV3}
startMarker="# BatchImportWithRefExample"
endMarker="# END BatchImportWithRefExample"
- language="py"
+ language="pyv3"
/>
@@ -360,7 +360,7 @@ You can batch create links from an object to another other object through cross-
text={TSCodeLegacy}
startMarker="// BatchImportWithRefExample"
endMarker="// END BatchImportWithRefExample"
- language="ts"
+ language="tsv2"
/>
@@ -423,7 +423,7 @@ To try the example code, download the sample data and create the sample input fi
text={PyCodeV3}
startMarker="# START JSON streaming"
endMarker="# END JSON streaming"
- language="py"
+ language="pyv3"
/>
@@ -456,7 +456,7 @@ To try the example code, download the sample data and create the sample input fi
text={PyCodeV3}
startMarker="# START CSV streaming"
endMarker="# END CSV streaming"
- language="py"
+ language="pyv3"
/>
diff --git a/developers/weaviate/manage-data/migrate.mdx b/developers/weaviate/manage-data/migrate.mdx
index b778168cbe..6944faa1e1 100644
--- a/developers/weaviate/manage-data/migrate.mdx
+++ b/developers/weaviate/manage-data/migrate.mdx
@@ -1,5 +1,6 @@
---
title: Migrate data
+description: Learn how to migrate data within Weaviate for easy data handling.
sidebar_position: 80
image: og/docs/howto.jpg
# tags: ['how-to', 'cursor']
@@ -12,9 +13,6 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PyCode from '!!raw-loader!/_includes/code/howto/manage-data.migrate.data.v4.py';
import PyCodeV3 from '!!raw-loader!/_includes/code/howto/manage-data.migrate.data.v3.py';
-
-## Overview
-
Follow these examples to migrate data manually when using a backup is not possible. They cover all permutations between:
- a single-tenancy collection (Collection), and
@@ -60,7 +58,7 @@ Create a collection (e.g. `WineReview`) at the target instance, matching the col
text={PyCodeV3}
startMarker="# START CreateCollectionCollectionToCollection"
endMarker="# END CreateCollectionCollectionToCollection"
- language="py"
+ language="pyv3"
/>
@@ -89,7 +87,7 @@ Migrate:
text={PyCodeV3}
startMarker="# START CollectionToCollection"
endMarker="# END CollectionToCollection"
- language="py"
+ language="pyv3"
/>
@@ -114,7 +112,7 @@ Create a collection (e.g. `WineReview`) at the target instance, matching the col
text={PyCodeV3}
startMarker="# START CreateCollectionCollectionToTenant"
endMarker="# END CreateCollectionCollectionToTenant"
- language="py"
+ language="pyv3"
/>
@@ -137,7 +135,7 @@ Add tenants at the target instance before adding data objects.
text={PyCodeV3}
startMarker="# START CreateTenants"
endMarker="# END CreateTenants"
- language="py"
+ language="pyv3"
/>
@@ -166,7 +164,7 @@ Migrate:
text={PyCodeV3}
startMarker="# START CollectionToTenant"
endMarker="# END CollectionToTenant"
- language="py"
+ language="pyv3"
/>
@@ -191,7 +189,7 @@ Create a collection (e.g. `WineReview`) at the target instance, matching the col
text={PyCodeV3}
startMarker="# START CreateCollectionTenantToCollection"
endMarker="# END CreateCollectionTenantToCollection"
- language="py"
+ language="pyv3"
/>
@@ -220,7 +218,7 @@ Migrate:
text={PyCodeV3}
startMarker="# START TenantToCollection"
endMarker="# END TenantToCollection"
- language="py"
+ language="pyv3"
/>
@@ -245,7 +243,7 @@ Create a collection (e.g. `WineReview`) at the target instance, matching the col
text={PyCodeV3}
startMarker="# START CreateCollectionTenantToTenant"
endMarker="# END CreateCollectionTenantToTenant"
- language="py"
+ language="pyv3"
/>
@@ -268,7 +266,7 @@ Add tenants at the target instance before adding data objects.
text={PyCodeV3}
startMarker="# START CreateTenants"
endMarker="# END CreateTenants"
- language="py"
+ language="pyv3"
/>
@@ -297,7 +295,7 @@ Migrate:
text={PyCodeV3}
startMarker="# START TenantToTenant"
endMarker="# END TenantToTenant"
- language="py"
+ language="pyv3"
/>
diff --git a/developers/weaviate/manage-data/multi-tenancy.md b/developers/weaviate/manage-data/multi-tenancy.md
index 7046a3e20f..f25118c590 100644
--- a/developers/weaviate/manage-data/multi-tenancy.md
+++ b/developers/weaviate/manage-data/multi-tenancy.md
@@ -51,7 +51,7 @@ Multi-tenancy is disabled by default. To enable multi-tenancy, set `multiTenancy
text={PyCodeV3}
startMarker="# START EnableMultiTenancy"
endMarker="# END EnableMultiTenancy"
- language="py"
+ language="pyv3"
/>
@@ -67,7 +67,7 @@ Multi-tenancy is disabled by default. To enable multi-tenancy, set `multiTenancy
text={TSCodeLegacy}
startMarker="// START EnableMultiTenancy"
endMarker="// END EnableMultiTenancy"
- language="ts"
+ language="tsv2"
/>
@@ -197,7 +197,7 @@ Tenant status is available from Weaviate `1.21` onwards.
text={PyCodeV3}
startMarker="# START AddTenantsToClass"
endMarker="# END AddTenantsToClass"
- language="py"
+ language="pyv3"
/>
@@ -215,7 +215,7 @@ Tenant status is available from Weaviate `1.21` onwards.
text={TSCodeLegacy}
startMarker="// START AddTenantsToClass"
endMarker="// END AddTenantsToClass"
- language="ts"
+ language="tsv2"
/>
@@ -259,7 +259,7 @@ This example lists the tenants in the `MultiTenancyCollection` collection:
text={PyCodeV3}
startMarker="# START ListTenants"
endMarker="# END ListTenants"
- language="py"
+ language="pyv3"
/>
@@ -277,7 +277,7 @@ This example lists the tenants in the `MultiTenancyCollection` collection:
text={TSCodeLegacy}
startMarker="// START ListTenants"
endMarker="// END ListTenants"
- language="ts"
+ language="tsv2"
/>
@@ -378,7 +378,7 @@ Deleting a tenant deletes all associated objects.
text={PyCodeV3}
startMarker="# START RemoveTenants"
endMarker="# END RemoveTenants"
- language="py"
+ language="pyv3"
/>
@@ -396,7 +396,7 @@ Deleting a tenant deletes all associated objects.
text={TSCodeLegacy}
startMarker="// START RemoveTenants"
endMarker="// END RemoveTenants"
- language="ts"
+ language="tsv2"
/>
@@ -468,7 +468,7 @@ Multi-tenancy collections require tenant name (e.g. `tenantA`) with each CRUD op
text={PyCodeV3}
startMarker="# START CreateMtObject"
endMarker="# END CreateMtObject"
- language="py"
+ language="pyv3"
/>
@@ -486,7 +486,7 @@ Multi-tenancy collections require tenant name (e.g. `tenantA`) with each CRUD op
text={TSCodeLegacy}
startMarker="// START CreateMtObject"
endMarker="// END CreateMtObject"
- language="ts"
+ language="tsv2"
/>
@@ -528,7 +528,7 @@ Multi-tenancy collections require the tenant name (e.g. `tenantA`) with each `Ge
text={PyCodeV3}
startMarker="# START Search"
endMarker="# END Search"
- language="py"
+ language="pyv3"
/>
@@ -546,7 +546,7 @@ Multi-tenancy collections require the tenant name (e.g. `tenantA`) with each `Ge
text={TSCodeLegacy}
startMarker="// START Search"
endMarker="// END Search"
- language="ts"
+ language="tsv2"
/>
@@ -592,7 +592,7 @@ Multi-tenancy collections require the tenant name (e.g. `tenantA`) when creating
text={PyCodeV3}
startMarker="# START AddCrossRef"
endMarker="# END AddCrossRef"
- language="py"
+ language="pyv3"
/>
@@ -610,7 +610,7 @@ Multi-tenancy collections require the tenant name (e.g. `tenantA`) when creating
text={TSCodeLegacy}
startMarker="// START AddCrossRef"
endMarker="// END AddCrossRef"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/manage-data/read-all-objects.mdx b/developers/weaviate/manage-data/read-all-objects.mdx
index d042e2e202..4658e777dd 100644
--- a/developers/weaviate/manage-data/read-all-objects.mdx
+++ b/developers/weaviate/manage-data/read-all-objects.mdx
@@ -44,7 +44,7 @@ The following code iterates through all objects, providing the properties and id
text={PyCodeV3}
startMarker="# START ReadAllProps"
endMarker="# END ReadAllProps"
- language="py"
+ language="pyv3"
/>
@@ -62,7 +62,7 @@ The following code iterates through all objects, providing the properties and id
text={TSCodeLegacy}
startMarker="// START ReadAllProps"
endMarker="// END ReadAllProps"
- language="ts"
+ language="tsv2"
/>
@@ -105,7 +105,7 @@ Read through all data including the vectors. (Also applicable where [named vecto
text={PyCodeV3}
startMarker="# START ReadAllVectors"
endMarker="# END ReadAllVectors"
- language="py"
+ language="pyv3"
/>
@@ -123,7 +123,7 @@ Read through all data including the vectors. (Also applicable where [named vecto
text={TSCodeLegacy}
startMarker="// START ReadAllVectors"
endMarker="// END ReadAllVectors"
- language="ts"
+ language="tsv2"
/>
@@ -152,7 +152,7 @@ For classes where [multi-tenancy](../concepts/data.md#multi-tenancy) is enabled,
text={PyCodeV3}
startMarker="# START ReadAllTenants"
endMarker="# END ReadAllTenants"
- language="py"
+ language="pyv3"
/>
@@ -170,7 +170,7 @@ For classes where [multi-tenancy](../concepts/data.md#multi-tenancy) is enabled,
text={TSCodeLegacy}
startMarker="// START ReadAllTenants"
endMarker="// END ReadAllTenants"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/manage-data/read.mdx b/developers/weaviate/manage-data/read.mdx
index af4b79755f..6afeece205 100644
--- a/developers/weaviate/manage-data/read.mdx
+++ b/developers/weaviate/manage-data/read.mdx
@@ -44,7 +44,7 @@ Use an ID to retrieve an object. If the id doesn't exist, Weaviate returns a 404
text={PyCodeV3}
startMarker="# ReadObject START"
endMarker="# ReadObject END"
- language="py"
+ language="pyv3"
/>
@@ -62,7 +62,7 @@ Use an ID to retrieve an object. If the id doesn't exist, Weaviate returns a 404
text={TSCodeLegacy}
startMarker="// ReadObject START"
endMarker="// ReadObject END"
- language="ts"
+ language="tsv2"
/>
@@ -105,7 +105,7 @@ Object vectors can be retrieved by specifying its return.
text={PyCodeV3}
startMarker="# ReadObjectWithVector START"
endMarker="# ReadObjectWithVector END"
- language="py"
+ language="pyv3"
/>
@@ -123,7 +123,7 @@ Object vectors can be retrieved by specifying its return.
text={TSCodeLegacy}
startMarker="// ReadObjectWithVector START"
endMarker="// ReadObjectWithVector END"
- language="ts"
+ language="tsv2"
/>
@@ -165,7 +165,7 @@ Where [named vectors](../config-refs/schema/multi-vector.md) are used, you can r
text={PyCodeV3}
startMarker="# ReadObjectNamedVectors START"
endMarker="# ReadObjectNamedVectors END"
- language="py"
+ language="pyv3"
/>
@@ -183,7 +183,7 @@ Where [named vectors](../config-refs/schema/multi-vector.md) are used, you can r
text={TSCodeLegacy}
startMarker="// ReadObjectNamedVectors START"
endMarker="// ReadObjectNamedVectors END"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/manage-data/tenant-states.mdx b/developers/weaviate/manage-data/tenant-states.mdx
index 99d6688459..bf69339750 100644
--- a/developers/weaviate/manage-data/tenant-states.mdx
+++ b/developers/weaviate/manage-data/tenant-states.mdx
@@ -1,5 +1,6 @@
---
title: Manage tenant states & temperature
+description: Manage tenant states in Weaviate to support multitenancy and secure data separation.
sidebar_position: 61
image: og/docs/configuration.jpg
---
@@ -10,8 +11,6 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl
import PyCode from '!!raw-loader!/_includes/code/howto/manage-data.multi-tenancy.py';
import TSCode from '!!raw-loader!/_includes/code/howto/manage-data.multi-tenancy.ts';
-## Overview
-
Storage resources are grouped into tiers. Each tier has different performance characteristics and costs:
diff --git a/developers/weaviate/manage-data/update.mdx b/developers/weaviate/manage-data/update.mdx
index 8c94b83fef..fa0ad59748 100644
--- a/developers/weaviate/manage-data/update.mdx
+++ b/developers/weaviate/manage-data/update.mdx
@@ -36,7 +36,7 @@ Weaviate allows partial or complete object updates.
## Update object properties
-To update a collection property, provide the collection name, the object id, and the properties to update.
+This operation replaces the entire value of the specified properties only, leaving the unspecified properties. Provide the collection name, the object id, and the properties to update.
If you update the value of a *previously vectorized* property, Weaviate re-vectorizes the object automatically. This also reindexes the updated object.
@@ -56,7 +56,7 @@ However, if you add a *new* property to your collection definition, Weaviate onl
text={PyCodeV3}
startMarker="# UpdateProps START"
endMarker="# UpdateProps END"
- language="py"
+ language="pyv3"
/>
@@ -72,7 +72,7 @@ However, if you add a *new* property to your collection definition, Weaviate onl
text={TSCodeLegacy}
startMarker="// UpdateProps START"
endMarker="// UpdateProps END"
- language="ts"
+ language="tsv2"
/>
@@ -111,7 +111,7 @@ The object vector can also be updated similarly to properties. For [named vector
text={PyCodeV3}
startMarker="# UpdateVector START"
endMarker="# UpdateVector END"
- language="py"
+ language="pyv3"
/>
@@ -162,7 +162,7 @@ The entire object can be replaced by providing the collection name, id and the n
text={PyCodeV3}
startMarker="# Replace START"
endMarker="# Replace END"
- language="py"
+ language="pyv3"
/>
@@ -180,7 +180,7 @@ The entire object can be replaced by providing the collection name, id and the n
text={TSCodeLegacy}
startMarker="// Replace START"
endMarker="// Replace END"
- language="ts"
+ language="tsv2"
/>
@@ -224,7 +224,7 @@ At object level, you can replace the object with a copy that has those propertie
text={PyCodeV3}
startMarker="# DelProps START"
endMarker="# DelProps END"
- language="py"
+ language="pyv3"
/>
@@ -242,7 +242,7 @@ At object level, you can replace the object with a copy that has those propertie
text={TSCodeLegacy}
startMarker="// DelProps START"
endMarker="// DelProps END"
- language="ts"
+ language="tsv2"
/>
diff --git a/developers/weaviate/model-providers/_includes/google-api-key-note.md b/developers/weaviate/model-providers/_includes/google-api-key-note.md
index 3d410d215b..47e45d8efd 100644
--- a/developers/weaviate/model-providers/_includes/google-api-key-note.md
+++ b/developers/weaviate/model-providers/_includes/google-api-key-note.md
@@ -1,9 +1,9 @@
API key headers
-Starting from `v1.25.1` and `v1.24.14`, there are separate headers `X-Google-Vertex-Api-Key` and `X-Google-Studio-Api-Key` for Vertex AI users and AI Studio respectively.
+From `v1.27.7`, `v1.26.12` and `v1.25.27`, `X-Goog-Vertex-Api-Key` and `X-Goog-Studio-Api-Key` headers are supported for Vertex AI users and AI Studio respectively. We recommend these headers for highest compatibility.
-Prior to Weaviate `v1.25.1` or `v1.24.14`, there was one header for both Vertex AI users and AI Studio, specified with either `X-Google-Api-Key` or `X-PaLM-Api-Key`. We recommend using the new headers for clarity and future compatibility.
+Consider `X-Google-Vertex-Api-Key`, `X-Google-Studio-Api-Key`, `X-Google-Api-Key` and `X-PaLM-Api-Key` deprecated.
diff --git a/developers/weaviate/model-providers/_includes/integration_jinaai_reranker.png b/developers/weaviate/model-providers/_includes/integration_jinaai_reranker.png
new file mode 100644
index 0000000000..f870101c77
Binary files /dev/null and b/developers/weaviate/model-providers/_includes/integration_jinaai_reranker.png differ
diff --git a/developers/weaviate/model-providers/_includes/integration_wes_embedding.png b/developers/weaviate/model-providers/_includes/integration_wes_embedding.png
new file mode 100644
index 0000000000..3aacecfcdd
Binary files /dev/null and b/developers/weaviate/model-providers/_includes/integration_wes_embedding.png differ
diff --git a/developers/weaviate/model-providers/_includes/integration_wes_embedding_search.png b/developers/weaviate/model-providers/_includes/integration_wes_embedding_search.png
new file mode 100644
index 0000000000..f2682c999b
Binary files /dev/null and b/developers/weaviate/model-providers/_includes/integration_wes_embedding_search.png differ
diff --git a/developers/weaviate/model-providers/_includes/provider.connect.py b/developers/weaviate/model-providers/_includes/provider.connect.py
index 5525acf4c3..d04123c34b 100644
--- a/developers/weaviate/model-providers/_includes/provider.connect.py
+++ b/developers/weaviate/model-providers/_includes/provider.connect.py
@@ -103,9 +103,9 @@
"X-Friendli-Baseurl": "https://inference.friendli.ai/dedicated",
# END FriendliDedicatedInstantiation
# START GoogleInstantiation # START GoogleVertexInstantiation
- "X-Google-Vertex-Api-Key": vertex_key,
+ "X-Goog-Vertex-Api-Key": vertex_key,
# START GoogleInstantiation # END GoogleVertexInstantiation
- "X-Google-Studio-Api-Key": studio_key,
+ "X-Goog-Studio-Api-Key": studio_key,
# END GoogleInstantiation
# START HuggingFaceInstantiation
"X-HuggingFace-Api-Key": huggingface_key,
diff --git a/developers/weaviate/model-providers/_includes/provider.connect.weaviate.py b/developers/weaviate/model-providers/_includes/provider.connect.weaviate.py
new file mode 100644
index 0000000000..0d0ac8b145
--- /dev/null
+++ b/developers/weaviate/model-providers/_includes/provider.connect.weaviate.py
@@ -0,0 +1,24 @@
+# START WeaviateInstantiation
+import weaviate
+from weaviate.classes.init import Auth
+import os
+
+# END WeaviateInstantiation
+
+weaviate_url = os.getenv("WEAVIATE_URL")
+weaviate_key = os.getenv("WEAVIATE_API_KEY")
+
+# START WeaviateInstantiation
+client = weaviate.connect_to_weaviate_cloud(
+ cluster_url=weaviate_url, # `weaviate_url`: your Weaviate URL
+ auth_credentials=Auth.api_key(weaviate_key), # `weaviate_key`: your Weaviate API key
+)
+# END WeaviateInstantiation
+
+
+# START WeaviateInstantiation
+
+# Work with Weaviate
+
+client.close()
+# END WeaviateInstantiation
diff --git a/developers/weaviate/model-providers/_includes/provider.connect.weaviate.ts b/developers/weaviate/model-providers/_includes/provider.connect.weaviate.ts
new file mode 100644
index 0000000000..75d9c65b4b
--- /dev/null
+++ b/developers/weaviate/model-providers/_includes/provider.connect.weaviate.ts
@@ -0,0 +1,18 @@
+// START WeaviateInstantiation
+import weaviate from 'weaviate-client'
+
+// END WeaviateInstantiation
+
+// START WeaviateInstantiation
+const client = await weaviate.connectToWeaviateCloud(
+ 'WEAVIATE_INSTANCE_URL', // Replace with your instance URL
+ {
+ authCredentials: new weaviate.ApiKey('WEAVIATE_INSTANCE_APIKEY'),
+ }
+)
+
+// Work with Weaviate
+
+client.close()
+// END WeaviateInstantiation
+
diff --git a/developers/weaviate/model-providers/_includes/provider.generative.ts b/developers/weaviate/model-providers/_includes/provider.generative.ts
index 5e07816bb9..062086f393 100644
--- a/developers/weaviate/model-providers/_includes/provider.generative.ts
+++ b/developers/weaviate/model-providers/_includes/provider.generative.ts
@@ -296,7 +296,7 @@ await client.collections.delete('DemoCollection');
await client.collections.create({
name: 'DemoCollection',
// highlight-start
- generative: weaviate.configure.generative.palm({
+ generative: weaviate.configure.generative.google({
projectId: '', // Required for Vertex AI
modelId: 'gemini-1.0-pro'
}),
@@ -309,7 +309,15 @@ await client.collections.create({
await client.collections.delete('DemoCollection');
// START BasicGenerativeGoogleStudio
-// Coming soon
+await client.collections.create({
+ name: 'DemoCollection',
+ // highlight-start
+ generative: weaviate.configure.generative.google({
+ modelId: 'gemini-pro'
+ }),
+ // highlight-end
+ // Additional parameters not shown
+});
// END BasicGenerativeGoogleStudio
// Clean up
@@ -319,7 +327,7 @@ await client.collections.delete('DemoCollection');
await client.collections.create({
name: 'DemoCollection',
// highlight-start
- generative: weaviate.configure.generative.palm({
+ generative: weaviate.configure.generative.google({
projectId: '', // Required for Vertex AI
// model_id='',
// api_endpoint='',
diff --git a/developers/weaviate/model-providers/_includes/provider.reranker.py b/developers/weaviate/model-providers/_includes/provider.reranker.py
index e9b5d01862..ddc224bf6a 100644
--- a/developers/weaviate/model-providers/_includes/provider.reranker.py
+++ b/developers/weaviate/model-providers/_includes/provider.reranker.py
@@ -60,6 +60,38 @@
# Clean up
client.collections.delete("DemoCollection")
+# START RerankerJinaAIBasic
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ reranker_config=Configure.Reranker.jinaai()
+ # highlight-end
+ # Additional parameters not shown
+)
+# END RerankerJinaAIBasic
+
+# Clean up
+client.collections.delete("DemoCollection")
+
+# START RerankerJinaAICustomModel
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ reranker_config=Configure.Reranker.jinaai(
+ model="jina-reranker-v2-base-multilingual"
+ )
+ # highlight-end
+ # Additional parameters not shown
+)
+# END RerankerJinaAICustomModel
+
+# Clean up
+client.collections.delete("DemoCollection")
+
# START RerankerVoyageAI
client.collections.create(
"DemoCollection",
diff --git a/developers/weaviate/model-providers/_includes/provider.reranker.ts b/developers/weaviate/model-providers/_includes/provider.reranker.ts
index 36824f9811..98c1230868 100644
--- a/developers/weaviate/model-providers/_includes/provider.reranker.ts
+++ b/developers/weaviate/model-providers/_includes/provider.reranker.ts
@@ -41,8 +41,35 @@ await client.collections.create({
});
// END RerankerCohereCustomModel
+// START RerankerJinaAIBasic
+await client.collections.create({
+ name: 'DemoCollection',
+ // highlight-start
+ reranker: weaviate.configure.reranker.jinaai(),
+ // highlight-end
+});
+// END RerankerJinaAIBasic
+
+// START RerankerJinaAICustomModel
+await client.collections.create({
+ name: 'DemoCollection',
+ // highlight-start
+ reranker: weaviate.configure.reranker.jinaai({
+ model: 'jina-reranker-v2-base-multilingual',
+ }),
+ // highlight-end
+});
+// END RerankerJinaAICustomModel
+
// START RerankerVoyageAI
-// Code example coming soon
+await client.collections.create({
+ name: 'DemoCollection',
+ // highlight-start
+ reranker: weaviate.configure.reranker.voyageAI({
+ model: 'rerank-lite-1',
+ }),
+ // highlight-end
+});
// END RerankerVoyageAI
// START RerankerQueryExample
diff --git a/developers/weaviate/model-providers/_includes/provider.vectorizer.py b/developers/weaviate/model-providers/_includes/provider.vectorizer.py
index b4a12a68f6..8e3d1142ec 100644
--- a/developers/weaviate/model-providers/_includes/provider.vectorizer.py
+++ b/developers/weaviate/model-providers/_includes/provider.vectorizer.py
@@ -148,6 +148,101 @@
# clean up
client.collections.delete("DemoCollection")
+# START BasicMMVectorizerCohere
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.multi2vec_cohere(
+ name="title_vector",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END BasicMMVectorizerCohere
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START MMVectorizerCohereCustomModel
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.text2vec_cohere(
+ name="title_vector",
+ model="embed-multilingual-v3.0",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END MMVectorizerCohereCustomModel
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START FullMMVectorizerCohere
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.text2vec_cohere(
+ name="title_vector",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ # Further options
+ # model="embed-multilingual-v3.0",
+ # truncate="END", # "NONE", "START" or "END"
+ # base_url=""
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END FullMMVectorizerCohere
+
+# clean up
+client.collections.delete("DemoCollection")
+
# START BasicVectorizerGoogleVertex
from weaviate.classes.config import Configure
@@ -366,8 +461,7 @@
Configure.NamedVectors.text2vec_jinaai(
name="title_vector",
source_properties=["title"],
- model="jina-embeddings-v3-small-en",
- dimensions=512, # e.g. 1024, 256, 64
+ model="jina-embeddings-v3",
)
],
# highlight-end
@@ -388,7 +482,8 @@
name="title_vector",
source_properties=["title"],
# Further options
- # model="jina-embeddings-v2-base-en"
+ # model="jina-embeddings-v3",
+ # dimensions=512, # e.g. 1024, 256, 64 (only applicable for some models)
)
],
# highlight-end
@@ -399,6 +494,99 @@
# clean up
client.collections.delete("DemoCollection")
+# START BasicMMVectorizerJinaAI
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.multi2vec_jinaai(
+ name="title_vector",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END BasicMMVectorizerJinaAI
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START MMVectorizerJinaCustomModel
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.multi2vec_jinaai(
+ name="title_vector",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ model="jina-clip-v2",
+ )
+ ],
+ # highlight-end
+)
+# END MMVectorizerJinaCustomModel
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START FullMMVectorizerJinaAI
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.multi2vec_jinaai(
+ name="title_vector",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ # Further options
+ # model="jina-clip-v2",
+ # dimensions=512, # Only applicable for some models (e.g. `jina-clip-v2`)
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END FullMMVectorizerJinaAI
+
+# clean up
+client.collections.delete("DemoCollection")
+
# START BasicVectorizerMistral
from weaviate.classes.config import Configure
@@ -574,7 +762,7 @@
# If using older model family e.g. `ada`
model="ada",
model_version="002",
- type="text"
+ type_="text"
)
],
# highlight-end
@@ -711,7 +899,7 @@
Configure.NamedVectors.text2vec_voyageai(
name="title_vector",
source_properties=["title"],
- model="voyage-code-2"
+ model="voyage-3-lite"
)
],
# highlight-end
@@ -746,6 +934,167 @@
# clean up
client.collections.delete("DemoCollection")
+# START BasicMMVectorizerVoyageAI
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.multi2vec_voyageai(
+ name="title_vector",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END BasicMMVectorizerVoyageAI
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START MMVectorizerVoyageAICustomModel
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.multi2vec_voyageai(
+ name="title_vector",
+ model="voyage-multimodal-3",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END MMVectorizerVoyageAICustomModel
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START FullMMVectorizerVoyageAI
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ properties=[
+ Property(name="title", data_type=DataType.TEXT),
+ Property(name="poster", data_type=DataType.BLOB),
+ ],
+ vectorizer_config=[
+ Configure.NamedVectors.multi2vec_voyageai(
+ name="title_vector",
+ # Define the fields to be used for the vectorization - using image_fields, text_fields
+ image_fields=[
+ Multi2VecField(name="poster", weight=0.9)
+ ],
+ text_fields=[
+ Multi2VecField(name="title", weight=0.1)
+ ],
+ # Further options
+ # model="voyage-multimodal-3",
+ # truncation="true", # or "false",
+ # output_encoding="base64", # or "null"
+ # base_url=""
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END FullMMVectorizerVoyageAI
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START BasicVectorizerWeaviate
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ vectorizer_config=[
+ Configure.NamedVectors.text2vec_weaviate(
+ name="title_vector",
+ source_properties=["title"]
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END BasicVectorizerWeaviate
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START VectorizerWeaviateCustomModel
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ vectorizer_config=[
+ Configure.NamedVectors.text2vec_weaviate(
+ name="title_vector",
+ source_properties=["title"],
+ model="arctic-embed-m-v1.5"
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END VectorizerWeaviateCustomModel
+
+# clean up
+client.collections.delete("DemoCollection")
+
+# START FullVectorizerWeaviate
+from weaviate.classes.config import Configure
+
+client.collections.create(
+ "DemoCollection",
+ # highlight-start
+ vectorizer_config=[
+ Configure.NamedVectors.text2vec_weaviate(
+ name="title_vector",
+ source_properties=["title"],
+ # Further options
+ # model="arctic-embed-m-v1.5",
+ # dimensions=256
+ # base_url="",
+ )
+ ],
+ # highlight-end
+ # Additional parameters not shown
+)
+# END FullVectorizerWeaviate
+
+# clean up
+client.collections.delete("DemoCollection")
+
# START BasicVectorizerTransformers
from weaviate.classes.config import Configure
@@ -1060,7 +1409,6 @@ def url_to_base64(url):
poster_b64 = url_to_base64(src_obj["poster_path"])
weaviate_obj = {
"title": src_obj["title"],
- "description": src_obj["description"],
"poster": poster_b64 # Add the image in base64 encoding
}
diff --git a/developers/weaviate/model-providers/_includes/provider.vectorizer.ts b/developers/weaviate/model-providers/_includes/provider.vectorizer.ts
index 87a4ac4233..f77385c42a 100644
--- a/developers/weaviate/model-providers/_includes/provider.vectorizer.ts
+++ b/developers/weaviate/model-providers/_includes/provider.vectorizer.ts
@@ -173,6 +173,116 @@ await client.collections.create({
// Clean up
await client.collections.delete('DemoCollection');
+// START BasicMMVectorizerCohere
+await client.collections.create({
+ name: "DemoCollection",
+ // highlight-start
+ properties: [
+ {
+ name: 'title',
+ dataType: weaviate.configure.dataType.TEXT,
+ },
+ {
+ name: 'poster',
+ dataType: weaviate.configure.dataType.BLOB,
+ },
+ ],
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecCohere({
+ name: "title_vector",
+ // Define the fields to be used for the vectorization - using imageFields, textFields
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }]
+ })],
+ // highlight-end
+ // Additional parameters not shown
+})
+// END BasicMMVectorizerCohere
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START MMVectorizerCohereCustomModel
+await client.collections.create({
+ name: "DemoCollection",
+ // highlight-start
+ properties: [
+ {
+ name: 'title',
+ dataType: weaviate.configure.dataType.TEXT,
+ },
+ {
+ name: 'poster',
+ dataType: weaviate.configure.dataType.BLOB,
+ },
+ ],
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecCohere({
+ name: "title_vector",
+ model: "embed-multilingual-v3.0",
+ // Define the fields to be used for the vectorization - using imageFields, textFields
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }]
+ })],
+ // highlight-end
+ // Additional parameters not shown
+})
+// END MMVectorizerCohereCustomModel
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START FullMMVectorizerCohere
+await client.collections.create({
+ name: "DemoCollection",
+ // highlight-start
+ properties: [
+ {
+ name: 'title',
+ dataType: weaviate.configure.dataType.TEXT,
+ },
+ {
+ name: 'poster',
+ dataType: weaviate.configure.dataType.BLOB,
+ },
+ ],
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecCohere({
+ name: "title_vector",
+ // Define the fields to be used for the vectorization - using imageFields, textFields
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }],
+ // Further options
+ // model: "embed-multilingual-v3.0",
+ // truncate: "END", // "NONE", "START" or "END"
+ // baseURL: ""
+ })],
+ // highlight-end
+ // Additional parameters not shown
+})
+// END FullMMVectorizerCohere
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
// START BasicVectorizerDatabricks
const databricksVectorizerEndpoint = process.env.DATABRICKS_VECTORIZER_ENDPOINT || ''; // If saved as an environment variable
@@ -211,7 +321,7 @@ await client.collections.create({
],
// highlight-start
vectorizers: [
- weaviate.configure.vectorizer.text2VecPalm({
+ weaviate.configure.vectorizer.text2VecGoogle({
name: 'title_vector',
sourceProperties: ['title'],
projectId: '',
@@ -238,7 +348,7 @@ await client.collections.create({
],
// highlight-start
vectorizers: [
- weaviate.configure.vectorizer.text2VecPalm({
+ weaviate.configure.vectorizer.text2VecGoogle({
name: 'title_vector',
sourceProperties: ['title'],
// (Optional) To manually set the model ID
@@ -264,7 +374,7 @@ await client.collections.create({
],
// highlight-start
vectorizers: [
- weaviate.configure.vectorizer.text2VecPalm({
+ weaviate.configure.vectorizer.text2VecGoogle({
name: 'title_vector',
sourceProperties: ['title'],
projectId: '', // Required for Vertex AI
@@ -294,7 +404,7 @@ await client.collections.create({
],
// highlight-start
vectorizers: [
- weaviate.configure.vectorizer.multi2VecPalm({
+ weaviate.configure.vectorizer.multi2VecGoogle({
name: 'title_vector',
location: '',
projectId: '',
@@ -339,7 +449,7 @@ await client.collections.create({
],
// highlight-start
vectorizers: [
- weaviate.configure.vectorizer.multi2VecPalm({
+ weaviate.configure.vectorizer.multi2VecGoogle({
name: 'title_vector',
projectId: '',
modelId: '',
@@ -464,7 +574,7 @@ await client.collections.create({
weaviate.configure.vectorizer.text2VecJina({
name: 'title_vector',
sourceProperties: ['title'],
- model: 'jina-embeddings-v2-small-en'
+ model: 'jina-embeddings-v3'
}),
],
// highlight-end
@@ -486,11 +596,11 @@ await client.collections.create({
// highlight-start
vectorizers: [
weaviate.configure.vectorizer.text2VecJina({
- name: 'title_vector',
- sourceProperties: ['title'],
- // model: 'jina-embeddings-v3-small-en'
- // dimensions: 512, // e.g. 1024, 256, 64 // Support for this parameter is coming soon
- },
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ // model: 'jina-embeddings-v3-small-en'
+ // dimensions: 512, // e.g. 1024, 256, 64 Support for this parameter is coming soon (Only applicable for some models)
+ },
),
],
// highlight-end
@@ -501,36 +611,240 @@ await client.collections.create({
// Clean up
await client.collections.delete('DemoCollection');
+// START BasicMMVectorizerJinaAI
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ {
+ name: 'poster',
+ dataType: 'blob' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecJinaAI({
+ name: 'title_vector',
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }]
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
+// END BasicMMVectorizerJinaAI
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START MMVectorizerJinaCustomModel
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ {
+ name: 'poster',
+ dataType: 'blob' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecJinaAI({
+ name: 'title_vector',
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }],
+ model: "jina-clip-v2"
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
+// END MMVectorizerJinaCustomModel
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START FullMMVectorizerJinaAI
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ {
+ name: 'poster',
+ dataType: 'blob' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecJinaAI({
+ name: 'title_vector',
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }],
+ // Further options
+ // model:"jina-clip-v2",
+
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
+// END FullMMVectorizerJinaAI
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
// START BasicVectorizerMistral
-// TS support & code example coming soon
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecMistral({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ }),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
// END BasicVectorizerMistral
// Clean up
await client.collections.delete('DemoCollection');
// START FullVectorizerMistral
-// TS support & code example coming soon
-// END FullVectorizerMistral
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecMistral({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ model: 'mistral-embed'
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});// END FullVectorizerMistral
// Clean up
await client.collections.delete('DemoCollection');
// START BasicVectorizerOctoAI
-// Code example coming soon
-// END BasicVectorizerOctoAI
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecOctoAI({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+}); // END BasicVectorizerOctoAI
// Clean up
await client.collections.delete('DemoCollection');
// START VectorizerOctoAICustomModel
-// Code example coming soon
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecOctoAI({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ model: "thenlper/gte-large",
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
// END VectorizerOctoAICustomModel
// Clean up
await client.collections.delete('DemoCollection');
// START FullVectorizerOctoAI
-// Code example coming soon
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecOctoAI({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ // model: "thenlper/gte-large",
+ // vectorizeCollectionName: true,
+ // baseURL: "https://text.octoai.run",
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
// END FullVectorizerOctoAI
// Clean up
@@ -550,7 +864,7 @@ await client.collections.create({
weaviate.configure.vectorizer.text2VecOpenAI({
name: 'title_vector',
sourceProperties: ['title'],
- },
+ },
),
],
// highlight-end
@@ -577,7 +891,7 @@ await client.collections.create({
sourceProperties: ['title'],
model: 'text-embedding-3-large',
dimensions: 1024
- },
+ },
),
],
// highlight-end
@@ -605,7 +919,7 @@ await client.collections.create({
model: 'ada',
modelVersion: '002',
type: 'text'
- },
+ },
),
],
// highlight-end
@@ -689,11 +1003,11 @@ await client.collections.create({
// highlight-start
vectorizers: [
weaviate.configure.vectorizer.text2VecAzureOpenAI({
- name: 'title_vector',
- sourceProperties: ['title'],
- resourceName: '',
- deploymentId: '',
- },
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ resourceName: '',
+ deploymentId: '',
+ },
),
],
// highlight-end
@@ -716,13 +1030,13 @@ await client.collections.create({
// highlight-start
vectorizers: [
weaviate.configure.vectorizer.text2VecAzureOpenAI({
- name: 'title_vector',
- sourceProperties: ['title'],
- resourceName: '',
- deploymentId: '',
- // // Further options
- // baseURL: ''
- },
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ resourceName: '',
+ deploymentId: '',
+ // // Further options
+ // baseURL: ''
+ },
),
],
// highlight-end
@@ -745,9 +1059,9 @@ await client.collections.create({
// highlight-start
vectorizers: [
weaviate.configure.vectorizer.text2VecVoyageAI({
- name: 'title_vector',
- sourceProperties: ['title'],
- },
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ },
),
],
// highlight-end
@@ -772,7 +1086,7 @@ await client.collections.create({
weaviate.configure.vectorizer.text2VecVoyageAI({
name: 'title_vector',
sourceProperties: ['title'],
- model: 'voyage-code-2',
+ model: 'voyage-3-lite',
}),
],
// highlight-end
@@ -795,19 +1109,209 @@ await client.collections.create({
// highlight-start
vectorizers: [
weaviate.configure.vectorizer.text2VecVoyageAI({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ // // Further options
+ // model: 'voyage-large-2',
+ // base_url: '',
+ // truncate: true
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
+// END FullVectorizerVoyageAI
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START BasicMMVectorizerVoyageAI
+await client.collections.create({
+ name: "DemoCollection",
+ // highlight-start
+ properties: [
+ {
+ name: 'title',
+ dataType: weaviate.configure.dataType.TEXT,
+ },
+ {
+ name: 'poster',
+ dataType: weaviate.configure.dataType.BLOB,
+ },
+ ],
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecVoyageAI({
+ name: "title_vector",
+ // Define the fields to be used for the vectorization - using imageFields, textFields
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }]
+ })],
+ // highlight-end
+ // Additional parameters not shown
+})// END BasicMMVectorizerVoyageAI
+
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START MMVectorizerVoyageAICustomModel
+await client.collections.create({
+ name: "DemoCollection",
+ // highlight-start
+ properties: [
+ {
+ name: 'title',
+ dataType: weaviate.configure.dataType.TEXT,
+ },
+ {
+ name: 'poster',
+ dataType: weaviate.configure.dataType.BLOB,
+ },
+ ],
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecVoyageAI({
+ name: "title_vector",
+ // Define the fields to be used for the vectorization - using imageFields, textFields
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }],
+ model: "voyage-multimodal-3",
+ })],
+ // highlight-end
+ // Additional parameters not shown
+})// END MMVectorizerVoyageAICustomModel
+
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START FullMMVectorizerVoyageAI
+await client.collections.create({
+ name: "DemoCollection",
+ // highlight-start
+ properties: [
+ {
+ name: 'title',
+ dataType: weaviate.configure.dataType.TEXT,
+ },
+ {
+ name: 'poster',
+ dataType: weaviate.configure.dataType.BLOB,
+ },
+ ],
+ vectorizers: [
+ weaviate.configure.vectorizer.multi2VecVoyageAI({
+ name: "title_vector",
+ // Define the fields to be used for the vectorization - using imageFields, textFields
+ imageFields: [{
+ name: "poster",
+ weight: 0.9
+ }],
+ textFields: [{
+ name: "title",
+ weight: 0.1
+ }],
+ // Further options
+ // model: "voyage-multimodal-3",
+ // truncation: "true", // or "false"
+ // outputEncoding: "base64" // or "null"
+ // baseURL: ""
+ })],
+ // highlight-end
+ // Additional parameters not shown
+})// END FullMMVectorizerVoyageAI
+
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START BasicVectorizerWeaviate
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecWeaviate({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ },
+ ),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
+// END BasicVectorizerWeaviate
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START VectorizerWeaviateCustomModel
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecWeaviate({
+ name: 'title_vector',
+ sourceProperties: ['title'],
+ model: 'arctic-embed-m-v1.5',
+ }),
+ ],
+ // highlight-end
+ // Additional parameters not shown
+});
+// END VectorizerWeaviateCustomModel
+
+// Clean up
+await client.collections.delete('DemoCollection');
+
+// START FullVectorizerWeaviate
+await client.collections.create({
+ name: 'DemoCollection',
+ properties: [
+ {
+ name: 'title',
+ dataType: 'text' as const,
+ },
+ ],
+ // highlight-start
+ vectorizers: [
+ weaviate.configure.vectorizer.text2VecWeaviate({
name: 'title_vector',
sourceProperties: ['title'],
// // Further options
- // model: 'voyage-large-2',
- // base_url: '',
- // truncate: true
+ // model: 'arctic-embed-m-v1.5',
+ // dimensions: 256,
+ // baseUrl: '',
},
),
],
// highlight-end
// Additional parameters not shown
});
-// END FullVectorizerVoyageAI
+// END FullVectorizerWeaviate
// Clean up
await client.collections.delete('DemoCollection');
@@ -826,7 +1330,7 @@ await client.collections.create({
weaviate.configure.vectorizer.text2VecTransformers({
name: 'title_vector',
sourceProperties: ['title'],
- },
+ },
),
],
// highlight-end
@@ -850,12 +1354,12 @@ await client.collections.create({
weaviate.configure.vectorizer.text2VecTransformers({
name: 'title_vector',
sourceProperties: ['title'],
- // // Further options
+ // // Further options
// poolingStrategy: 'masked_mean',
// inferenceUrl: '', // For when using multiple inference containers
// passageInferenceUrl: ``, // For when using DPR models
// queryInferenceUrl: ``, // For when using DPR models
- },
+ },
),
],
// highlight-end
@@ -878,7 +1382,7 @@ await client.collections.create({
// highlight-end
// END BasicVectorizerOllama
properties: [
- {name: 'title', dataType: 'text'}
+ { name: 'title', dataType: 'text' }
],
// START BasicVectorizerOllama
// Additional parameters not shown
@@ -901,7 +1405,7 @@ await client.collections.create({
// highlight-end
// END BasicVectorizerGPT4All
properties: [
- {name: 'title', dataType: 'text'}
+ { name: 'title', dataType: 'text' }
],
// START BasicVectorizerGPT4All
// Additional parameters not shown
@@ -924,7 +1428,7 @@ await client.collections.create({
// highlight-end
// END FullVectorizerGPT4All
properties: [
- {name: 'title', dataType: 'text'}
+ { name: 'title', dataType: 'text' }
],
// START FullVectorizerGPT4All
// Additional parameters not shown
@@ -1102,11 +1606,11 @@ await client.collections.delete('DemoCollection');
await client.collections.delete('DemoCollection');
let srcObjects = [
- {title: "The Shawshank Redemption", description: ""},
- {title: "The Godfather", description: ""},
- {title: "The Dark Knight", description: ""},
- {title: "Jingle All the Way", description: ""},
- {title: "A Christmas Carol", description: ""},
+ { title: "The Shawshank Redemption", description: "" },
+ { title: "The Godfather", description: "" },
+ { title: "The Dark Knight", description: "" },
+ { title: "Jingle All the Way", description: "" },
+ { title: "A Christmas Carol", description: "" },
];
// START BatchImportExample // START NearTextExample // START HybridExample // START MMBatchImportExample
@@ -1131,11 +1635,11 @@ console.log(response);
// END BatchImportExample
let mmSrcObjects = [
- {title: "The Shawshank Redemption", description: "", poster: ""},
- {title: "The Godfather", description: "", poster: ""},
- {title: "The Dark Knight", description: "", poster: ""},
- {title: "Jingle All the Way", description: "", poster: ""},
- {title: "A Christmas Carol", description: "", poster: ""},
+ { title: "The Shawshank Redemption", description: "", poster: "" },
+ { title: "The Godfather", description: "", poster: "" },
+ { title: "The Dark Knight", description: "", poster: "" },
+ { title: "Jingle All the Way", description: "", poster: "" },
+ { title: "A Christmas Carol", description: "", poster: "" },
];
// START MMBatchImportExample
diff --git a/developers/weaviate/model-providers/anthropic/generative.md b/developers/weaviate/model-providers/anthropic/generative.md
index a5cacab282..d3b0a9de0b 100644
--- a/developers/weaviate/model-providers/anthropic/generative.md
+++ b/developers/weaviate/model-providers/anthropic/generative.md
@@ -78,7 +78,11 @@ Provide the API key to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use an Anthropic generative model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use an Anthropic generative model:
diff --git a/developers/weaviate/model-providers/anthropic/index.md b/developers/weaviate/model-providers/anthropic/index.md
index c4d8de68dd..d61aa39ebc 100644
--- a/developers/weaviate/model-providers/anthropic/index.md
+++ b/developers/weaviate/model-providers/anthropic/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_anthropic.jpg
-Anthropic offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Anthropic's APIs, allowing users to leverage Anthropic's models directly within the Weaviate database.
+Anthropic offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Anthropic's APIs, allowing users to leverage Anthropic's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -19,7 +19,7 @@ These integrations empower developers to build sophisticated AI-driven applicati
Anthropic's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Anthropic's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Anthropic's generative AI models to generate personalized and context-aware responses.
[Anthropic generative AI integration page](./generative.md)
@@ -37,12 +37,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/anyscale/generative.md b/developers/weaviate/model-providers/anyscale/generative.md
index c4a841a96d..44be94504f 100644
--- a/developers/weaviate/model-providers/anyscale/generative.md
+++ b/developers/weaviate/model-providers/anyscale/generative.md
@@ -78,7 +78,11 @@ Provide the API key to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use an Anyscale generative model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use an Anyscale generative model:
diff --git a/developers/weaviate/model-providers/anyscale/index.md b/developers/weaviate/model-providers/anyscale/index.md
index 5001048c90..823c05a190 100644
--- a/developers/weaviate/model-providers/anyscale/index.md
+++ b/developers/weaviate/model-providers/anyscale/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_anyscale.jpg
-Anyscale offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Anyscale's Endpoints APIs, allowing users to leverage Anyscale's models directly within the Weaviate database.
+Anyscale offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Anyscale's Endpoints APIs, allowing users to leverage Anyscale's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -19,7 +19,7 @@ These integrations empower developers to build sophisticated AI-driven applicati
Anyscale's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Anyscale's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Anyscale's generative AI models to generate personalized and context-aware responses.
[Anyscale generative AI integration page](./generative.md)
@@ -37,12 +37,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/aws/embeddings.md b/developers/weaviate/model-providers/aws/embeddings.md
index cfc8f3c56e..db4bb86622 100644
--- a/developers/weaviate/model-providers/aws/embeddings.md
+++ b/developers/weaviate/model-providers/aws/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: "Weaviate's integration with AWS's SageMaker and Bedrock APIs allows you to access their models' capabilities directly from Weaviate."
sidebar_position: 20
image: og/docs/integrations/provider_integrations_aws.jpg
# tags: ['model providers', 'aws', 'embeddings']
@@ -13,8 +14,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with AWS's [SageMaker](https://aws.amazon.com/sagemaker/) and [Bedrock](https://aws.amazon.com/bedrock/) APIs allows you to access their models' capabilities directly from Weaviate.
@@ -74,6 +77,15 @@ Provide the API credentials to Weaviate using one of the following methods:
/>
+
+
+
+
### AWS model access
@@ -117,6 +129,15 @@ For Bedrock, you must provide the model name in the vectorizer configuration.
/>
+
+
+
+
### SageMaker
@@ -142,6 +163,15 @@ For SageMaker, you must provide the endpoint address in the vectorizer configura
/>
+
+
+
+
import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
@@ -181,6 +211,15 @@ The AWS region setting is required for all AWS integrations.
/>
+
+
+
+
## Data import
@@ -207,6 +246,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -245,6 +293,15 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
### Hybrid search
@@ -277,6 +334,15 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
## References
diff --git a/developers/weaviate/model-providers/aws/generative.md b/developers/weaviate/model-providers/aws/generative.md
index 48f0fc08ec..e71b8f2277 100644
--- a/developers/weaviate/model-providers/aws/generative.md
+++ b/developers/weaviate/model-providers/aws/generative.md
@@ -1,5 +1,6 @@
---
title: Generative AI
+description: AWS Generative Model Provider
sidebar_position: 50
image: og/docs/integrations/provider_integrations_aws.jpg
# tags: ['model providers', 'aws', 'generative', 'rag']
@@ -90,7 +91,11 @@ To use a model via [SageMaker](https://aws.amazon.com/sagemaker/), you must have
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use an AWS generative model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use an AWS generative model:
### Bedrock
diff --git a/developers/weaviate/model-providers/aws/index.md b/developers/weaviate/model-providers/aws/index.md
index 1ad4c7c594..e14470823c 100644
--- a/developers/weaviate/model-providers/aws/index.md
+++ b/developers/weaviate/model-providers/aws/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_aws.jpg
-AWS offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with AWS's APIs, allowing users to leverage AWS's models directly within the Weaviate database.
+AWS offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with AWS's APIs, allowing users to leverage AWS's models directly from the Weaviate database.
Weaviate integrates with both AWS [Sagemaker](https://aws.amazon.com/sagemaker/) and [Bedrock](https://aws.amazon.com/bedrock/).
@@ -35,7 +35,7 @@ AWS's embedding models transform text data into high-dimensional vector represen
AWS's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with AWS's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with AWS's generative AI models to generate personalized and context-aware responses.
[AWS generative AI integration page](./generative.md)
@@ -54,12 +54,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/cohere/embeddings-multimodal.md b/developers/weaviate/model-providers/cohere/embeddings-multimodal.md
new file mode 100644
index 0000000000..b598dfbf3b
--- /dev/null
+++ b/developers/weaviate/model-providers/cohere/embeddings-multimodal.md
@@ -0,0 +1,328 @@
+---
+title: Multimodal Embeddings
+description: "Weaviate's integration with Cohere's APIs allows you to access their models' capabilities directly from Weaviate."
+sidebar_position: 25
+image: og/docs/integrations/provider_integrations_cohere.jpg
+# tags: ['model providers', 'cohere', 'embeddings']
+---
+
+# Cohere Multimodal Embeddings with Weaviate
+
+:::info Added in `1.25.25`, `1.26.9` and `v1.27.2`
+:::
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
+import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
+import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+
+Weaviate's integration with Cohere's APIs allows you to access their models' capabilities directly from Weaviate.
+
+[Configure a Weaviate vector index](#configure-the-vectorizer) to use a Cohere embedding model, and Weaviate will generate embeddings for various operations using the specified model and your Cohere API key. This feature is called the *vectorizer*.
+
+At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts text queries into embeddings. [Multimodal search operations](#vector-near-media-search) are also supported.
+
+
+
+## Requirements
+
+### Weaviate configuration
+
+Your Weaviate instance must be configured with the Cohere vectorizer integration (`multi2vec-cohere`) module.
+
+
+ For Weaviate Cloud (WCD) users
+
+This integration is enabled by default on Weaviate Cloud (WCD) serverless instances.
+
+
+
+
+ For self-hosted users
+
+- Check the [cluster metadata](../../config-refs/meta.md) to verify if the module is enabled.
+- Follow the [how-to configure modules](../../configuration/modules.md) guide to enable the module in Weaviate.
+
+
+
+### API credentials
+
+You must provide a valid Cohere API key to Weaviate for this integration. Go to [Cohere](https://cohere.com/) to sign up and obtain an API key.
+
+Provide the API key to Weaviate using one of the following methods:
+
+- Set the `COHERE_APIKEY` environment variable that is available to Weaviate.
+- Provide the API key at runtime, as shown in the examples below.
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Configure the vectorizer
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a Cohere embedding model:
+
+
+
+
+
+
+
+
+
+
+
+
+### Select a model
+
+You can specify one of the [available models](#available-models) for the vectorizer to use, as shown in the following configuration example.
+
+
+
+
+
+
+
+
+
+
+
+
+You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
+
+import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
+
+
+ Vectorization behavior
+
+
+
+
+
+### Vectorizer parameters
+
+The following examples show how to configure Cohere-specific options.
+
+
+
+
+
+
+
+
+
+
+
+
+For further details on model parameters, see the [Cohere API documentation](https://docs.cohere.com/reference/embed).
+
+## Data import
+
+After configuring the vectorizer, [import data](../../manage-data/import.mdx) into Weaviate. Weaviate generates embeddings for text objects using the specified model.
+
+
+
+
+
+
+
+
+
+
+
+
+
+:::tip Re-use existing vectors
+If you already have a compatible model vector available, you can provide it directly to Weaviate. This can be useful if you have already generated embeddings using the same model and want to use them in Weaviate, such as when migrating data from another system.
+:::
+
+## Searches
+
+Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified Cohere model.
+
+
+
+### Vector (near text) search
+
+When you perform a [vector search](../../search/similarity.md#search-with-text), Weaviate converts the text query into an embedding using the specified model and returns the most similar objects from the database.
+
+The query below returns the `n` most similar objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Hybrid search
+
+:::info What is a hybrid search?
+A hybrid search performs a vector search and a keyword (BM25) search, before [combining the results](../../search/hybrid.md#change-the-ranking-method) to return the best matching objects from the database.
+:::
+
+When you perform a [hybrid search](../../search/hybrid.md), Weaviate converts the text query into an embedding using the specified model and returns the best scoring objects from the database.
+
+The query below returns the `n` best scoring objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Vector (near media) search
+
+When you perform a media search such as a [near image search](../../search/similarity.md#search-with-image), Weaviate converts the query into an embedding using the specified model and returns the most similar objects from the database.
+
+To perform a near media search such as near image search, convert the media query into a base64 string and pass it to the search query.
+
+The query below returns the `n` most similar objects to the input image from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+## References
+
+### Available models
+
+- `embed-multilingual-v3.0` (Default)
+- `embed-multilingual-light-v3.0`
+- `embed-english-v3.0`
+- `embed-english-light-v3.0`
+
+## Further resources
+
+### Other integrations
+
+- [Cohere text embedding models + Weaviate](./embeddings.md).
+- [Cohere generative models + Weaviate](./generative.md).
+- [Cohere reranker models + Weaviate](./reranker.md).
+
+### Code examples
+
+Once the integrations are configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples:
+
+- The [how-to: manage data](../../manage-data/index.md) guides show how to perform data operations (i.e. create, update, delete).
+- The [how-to: search](../../search/index.md) guides show how to perform search operations (i.e. vector, keyword, hybrid) as well as retrieval augmented generation.
+
+### External resources
+
+- Cohere [Embed API documentation](https://docs.cohere.com/reference/embed)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/model-providers/cohere/embeddings.md b/developers/weaviate/model-providers/cohere/embeddings.md
index c59f770d41..6ca553cc97 100644
--- a/developers/weaviate/model-providers/cohere/embeddings.md
+++ b/developers/weaviate/model-providers/cohere/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: "Weaviate's integration with Cohere's APIs allows you to access their models' capabilities directly from Weaviate."
sidebar_position: 20
image: og/docs/integrations/provider_integrations_cohere.jpg
# tags: ['model providers', 'cohere', 'embeddings']
@@ -13,8 +14,13 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
+import JavaConnect from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/Connect.java';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageCohereTextEmbeddings.java';
+import JavaImportQueries from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ImportAndQueries.java';
Weaviate's integration with Cohere's APIs allows you to access their models' capabilities directly from Weaviate.
@@ -56,21 +62,47 @@ Provide the API key to Weaviate using one of the following methods:
-
+
-
+
+
+
+
+
+
+
+
+
@@ -99,6 +131,24 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
+
+
+
+
### Select a model
@@ -124,6 +174,24 @@ You can specify one of the [available models](#available-models) for the vectori
/>
+
+
+
+
+
+
+
+
You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
@@ -160,6 +228,24 @@ The following examples show how to configure Cohere-specific options.
/>
+
+
+
+
+
+
+
+
For further details on model parameters, see the [Cohere API documentation](https://docs.cohere.com/reference/embed).
@@ -188,6 +274,24 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
+
+
+
+
:::tip Re-use existing vectors
@@ -226,6 +330,24 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
+
+
+
+
### Hybrid search
@@ -258,6 +380,23 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
+
+
+
## References
@@ -287,8 +426,9 @@ The following models are available, but deprecated:
### Other integrations
-- [Cohere generative models + Weaviate](./generative.md).
-- [Cohere reranker models + Weaviate](./reranker.md).
+- [Cohere multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md)
+- [Cohere generative models + Weaviate](./generative.md)
+- [Cohere reranker models + Weaviate](./reranker.md)
### Code examples
diff --git a/developers/weaviate/model-providers/cohere/generative.md b/developers/weaviate/model-providers/cohere/generative.md
index 7b969e614c..6464868823 100644
--- a/developers/weaviate/model-providers/cohere/generative.md
+++ b/developers/weaviate/model-providers/cohere/generative.md
@@ -1,5 +1,6 @@
---
title: Generative AI
+description: Cohere Generative Model Provider
sidebar_position: 50
image: og/docs/integrations/provider_integrations_cohere.jpg
# tags: ['model providers', 'cohere', 'generative', 'rag']
@@ -78,7 +79,11 @@ Provide the API key to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use a Cohere generative AI model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use a Cohere generative AI model:
@@ -241,7 +246,8 @@ In other words, when you have `n` search results, the generative model generates
### Other integrations
-- [Cohere embedding models + Weaviate](./embeddings.md).
+- [Cohere text embedding models + Weaviate](./embeddings.md).
+- [Cohere multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md)
- [Cohere reranker models + Weaviate](./reranker.md).
### Code examples
diff --git a/developers/weaviate/model-providers/cohere/index.md b/developers/weaviate/model-providers/cohere/index.md
index 5b6c7a44a1..1bd30c8f80 100644
--- a/developers/weaviate/model-providers/cohere/index.md
+++ b/developers/weaviate/model-providers/cohere/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_cohere.jpg
-Cohere offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Cohere's APIs, allowing users to leverage Cohere's models directly within the Weaviate database.
+Cohere offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Cohere's APIs, allowing users to leverage Cohere's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -29,7 +29,7 @@ Cohere's embedding models transform text data into high-dimensional vector repre
Cohere's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Cohere's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Cohere's generative AI models to generate personalized and context-aware responses.
[Cohere generative AI integration page](./generative.md)
@@ -59,12 +59,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Generative AI](./generative.md)
- [Reranker](./reranker.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/cohere/reranker.md b/developers/weaviate/model-providers/cohere/reranker.md
index dc5cbad6c0..1f7eede2d4 100644
--- a/developers/weaviate/model-providers/cohere/reranker.md
+++ b/developers/weaviate/model-providers/cohere/reranker.md
@@ -1,5 +1,6 @@
---
title: Reranker
+description: Cohere Reranker Model Provider
sidebar_position: 70
image: og/docs/integrations/provider_integrations_cohere.jpg
# tags: ['model providers', 'cohere', 'reranking']
@@ -78,6 +79,10 @@ Provide the API key to Weaviate using one of the following methods:
## Configure the reranker
+import MutableRerankerConfig from '/_includes/mutable-reranker-config.md';
+
+
+
Configure a Weaviate collection to use a Cohere reranker model as follows:
@@ -181,7 +186,8 @@ For further details on model parameters, see the [Cohere API documentation](http
### Other integrations
-- [Cohere embedding models + Weaviate](./embeddings.md).
+- [Cohere text embedding models + Weaviate](./embeddings.md).
+- [Cohere multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md)
- [Cohere generative models + Weaviate](./generative.md).
### Code examples
diff --git a/developers/weaviate/model-providers/databricks/embeddings.md b/developers/weaviate/model-providers/databricks/embeddings.md
index d42ff5c9ab..f0845c4f50 100644
--- a/developers/weaviate/model-providers/databricks/embeddings.md
+++ b/developers/weaviate/model-providers/databricks/embeddings.md
@@ -15,8 +15,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with Databricks' APIs allows you to access models hosted on their platform directly from Weaviate.
@@ -76,6 +78,15 @@ Provide the Dataricks token to Weaviate using one of the following methods:
/>
+
+
+
+
## Configure the vectorizer
@@ -101,6 +112,15 @@ Provide the Dataricks token to Weaviate using one of the following methods:
/>
+
+
+
+
This will configure Weaviate to use the vectorizer served through the endpoint you specify.
@@ -135,6 +155,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
language="ts"
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -173,6 +202,14 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
### Hybrid search
@@ -204,6 +241,15 @@ The query below returns the `n` best scoring objects from the database, set by `
language="ts"
/>
+
+
+
+
## References
diff --git a/developers/weaviate/model-providers/databricks/generative.md b/developers/weaviate/model-providers/databricks/generative.md
index e800ae391a..8d9b366916 100644
--- a/developers/weaviate/model-providers/databricks/generative.md
+++ b/developers/weaviate/model-providers/databricks/generative.md
@@ -79,7 +79,11 @@ Provide the Dataricks token to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate collection](../../manage-data/collections.mdx#specify-a-generative-module) to use a Databricks generative AI endpoint as follows:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate collection](../../manage-data/collections.mdx#specify-a-generative-model-integration) to use a Databricks generative AI endpoint as follows:
diff --git a/developers/weaviate/model-providers/databricks/index.md b/developers/weaviate/model-providers/databricks/index.md
index d5c0dfe3d7..ed1cfbf100 100644
--- a/developers/weaviate/model-providers/databricks/index.md
+++ b/developers/weaviate/model-providers/databricks/index.md
@@ -10,7 +10,7 @@ image: og/docs/integrations/provider_integrations_databricks.jpg
:::info Added in `v1.26.3`
:::
-Databricks offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Databricks' Foundation Model APIs, allowing users to leverage Databricks' models directly within the Weaviate database.
+Databricks offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Databricks' Foundation Model APIs, allowing users to leverage Databricks' models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -32,7 +32,7 @@ Databricks' embedding models transform text data into high-dimensional vector re
Databrick' generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate' efficient storage and fast retrieval capabilities with Databrick' generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate' efficient storage and fast retrieval capabilities with Databrick' generative AI models to generate personalized and context-aware responses.
[Databricks generative AI integration page](./generative.md)
@@ -51,12 +51,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/friendliai/generative.md b/developers/weaviate/model-providers/friendliai/generative.md
index af05698eb8..3f9587ddd1 100644
--- a/developers/weaviate/model-providers/friendliai/generative.md
+++ b/developers/weaviate/model-providers/friendliai/generative.md
@@ -81,7 +81,11 @@ Provide the Friendli token to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use a FriendliAI generative AI model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use a FriendliAI generative AI model:
diff --git a/developers/weaviate/model-providers/friendliai/index.md b/developers/weaviate/model-providers/friendliai/index.md
index aab9692c75..d9b0defbff 100644
--- a/developers/weaviate/model-providers/friendliai/index.md
+++ b/developers/weaviate/model-providers/friendliai/index.md
@@ -22,7 +22,7 @@ FriendliAI integration empowers developers to build sophisticated AI-driven appl
FriendliAI's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform Retrieval Augmented Generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with FriendliAI's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform Retrieval Augmented Generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with FriendliAI's generative AI models to generate personalized and context-aware responses.
Visit [FriendliAI generative AI integration page](./generative.md) for more information on our integrations with FriendliAI.
diff --git a/developers/weaviate/model-providers/google/embeddings-multimodal.md b/developers/weaviate/model-providers/google/embeddings-multimodal.md
index cbdf885105..a74a911481 100644
--- a/developers/weaviate/model-providers/google/embeddings-multimodal.md
+++ b/developers/weaviate/model-providers/google/embeddings-multimodal.md
@@ -5,7 +5,7 @@ image: og/docs/integrations/provider_integrations_google.jpg
# tags: ['model providers', 'google', 'embeddings']
---
-# Google AI Multimodal Embeddings with Weaviate
+# Google Multimodal Embeddings with Weaviate
import Tabs from '@theme/Tabs';
@@ -22,9 +22,9 @@ Weaviate's integration with [Google Vertex AI](https://cloud.google.com/vertex-a
Multimodal embeddings are currently not available to Google AI Studio users.
:::
-[Configure a Weaviate vector index](#configure-the-vectorizer) to use a Google AI embedding model, and Weaviate will generate embeddings for various operations using the specified model and your Google AI API key. This feature is called the *vectorizer*.
+[Configure a Weaviate vector index](#configure-the-vectorizer) to use a Google embedding model, and Weaviate will generate embeddings for various operations using the specified model and your Google API key. This feature is called the *vectorizer*.
-At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts queries of one or more modalities into embeddings.
+At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts queries of one or more modalities into embeddings. [Multimodal search operations](#vector-near-media-search) are also supported.
@@ -32,7 +32,7 @@ At [import time](#data-import), Weaviate generates multimodal object embeddings
### Weaviate configuration
-Your Weaviate instance must be configured with the Google AI vectorizer integration (`multi2vec-google`) module.
+Your Weaviate instance must be configured with the Google multimodal vectorizer integration (`multi2vec-google`) module.
:::info Module name change
`multi2vec-google` was called `multi2vec-palm` in Weaviate versions prior to `v1.27`.
@@ -115,7 +115,7 @@ import ApiKeyNote from '../_includes/google-api-key-note.md';
## Configure the vectorizer
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a Google AI embedding model:
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a Google embedding model:
@@ -153,7 +153,7 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
### Vectorizer parameters
-The following examples show how to configure Google AI-specific options.
+The following examples show how to configure Google-specific options.
- `location` (Required): e.g. `"us-central1"`
- `projectId` (Only required if using Vertex AI): e.g. `cloud-large-language-models`
@@ -214,7 +214,7 @@ If you already have a compatible model vector available, you can provide it dire
## Searches
-Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified Google AI model.
+Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified Google model.
@@ -318,7 +318,8 @@ The query below returns the `n` most similar objects to the input image from the
### Other integrations
-- [Google AI generative models + Weaviate](./generative.md).
+- [Google text embedding models + Weaviate](./embeddings.md)
+- [Google AI generative models + Weaviate](./generative.md)
### Code examples
diff --git a/developers/weaviate/model-providers/google/embeddings.md b/developers/weaviate/model-providers/google/embeddings.md
index 67993027ff..b238fe8f8f 100644
--- a/developers/weaviate/model-providers/google/embeddings.md
+++ b/developers/weaviate/model-providers/google/embeddings.md
@@ -1,11 +1,12 @@
---
title: Text Embeddings
+description: "Weaviate's integration with Google AI Studio and Google Vertex AI APIs allows you to access their models' capabilities directly from Weaviate."
sidebar_position: 20
image: og/docs/integrations/provider_integrations_google.jpg
# tags: ['model providers', 'google', 'embeddings']
---
-# Google AI Text Embeddings with Weaviate
+# Google Text Embeddings with Weaviate
import Tabs from '@theme/Tabs';
@@ -13,12 +14,14 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with [Google AI Studio](https://ai.google.dev/?utm_source=weaviate&utm_medium=referral&utm_campaign=partnerships&utm_content=) and [Google Vertex AI](https://cloud.google.com/vertex-ai) APIs allows you to access their models' capabilities directly from Weaviate.
-[Configure a Weaviate vector index](#configure-the-vectorizer) to use a Google AI embedding model, and Weaviate will generate embeddings for various operations using the specified model and your Google AI API key. This feature is called the *vectorizer*.
+[Configure a Weaviate vector index](#configure-the-vectorizer) to use a Google embedding model, and Weaviate will generate embeddings for various operations using the specified model and your Google API key. This feature is called the *vectorizer*.
At [import time](#data-import), Weaviate generates text object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts text queries into embeddings.
@@ -32,7 +35,7 @@ At the time of writing (November 2023), AI Studio is not available in all region
### Weaviate configuration
-Your Weaviate instance must be configured with the Google AI vectorizer integration (`text2vec-google`) module.
+Your Weaviate instance must be configured with the Google vectorizer integration (`text2vec-google`) module.
:::info Module name change
`text2vec-google` was called `text2vec-palm` in Weaviate versions prior to `v1.27`.
@@ -113,11 +116,20 @@ import ApiKeyNote from '../_includes/google-api-key-note.md';
/>
+
+
+
+
## Configure the vectorizer
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a Google AI embedding model:
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a Google embedding model:
Note that the required parameters differ between Vertex AI and AI Studio.
@@ -146,6 +158,15 @@ Vertex AI users must provide the Google Cloud project ID in the vectorizer confi
/>
+
+
+
+
### AI Studio
@@ -169,6 +190,15 @@ Vertex AI users must provide the Google Cloud project ID in the vectorizer confi
/>
+
+
+
+
import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
@@ -182,7 +212,7 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
### Vectorizer parameters
-The following examples show how to configure Google AI-specific options.
+The following examples show how to configure Google-specific options.
- `projectId` (Only required if using Vertex AI): e.g. `cloud-large-language-models`
- `apiEndpoint` (Optional): e.g. `us-central1-aiplatform.googleapis.com`
@@ -208,6 +238,15 @@ The following examples show how to configure Google AI-specific options.
/>
+
+
+
+
## Data import
@@ -234,6 +273,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -242,7 +290,7 @@ If you already have a compatible model vector available, you can provide it dire
## Searches
-Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified Google AI model.
+Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified Google model.
@@ -272,6 +320,15 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
### Hybrid search
@@ -304,6 +361,15 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
## References
@@ -328,7 +394,7 @@ AI Studio:
### Other integrations
-- [Google AI generative models + Weaviate](./generative.md).
+- [Google generative models + Weaviate](./generative.md).
### Code examples
diff --git a/developers/weaviate/model-providers/google/generative.md b/developers/weaviate/model-providers/google/generative.md
index a3742bb702..ee4c4a78c5 100644
--- a/developers/weaviate/model-providers/google/generative.md
+++ b/developers/weaviate/model-providers/google/generative.md
@@ -1,11 +1,12 @@
---
title: Generative AI
+description: Google Generative Model Provider
sidebar_position: 50
image: og/docs/integrations/provider_integrations_google.jpg
# tags: ['model providers', 'google', 'generative', 'rag']
---
-# Google AI Generative AI with Weaviate
+# Google Generative AI with Weaviate
import Tabs from '@theme/Tabs';
@@ -18,9 +19,9 @@ import TSCode from '!!raw-loader!../_includes/provider.generative.ts';
Weaviate's integration with [Google AI Studio](https://ai.google.dev/?utm_source=weaviate&utm_medium=referral&utm_campaign=partnerships&utm_content=) and [Google Vertex AI](https://cloud.google.com/vertex-ai) APIs allows you to access their models' capabilities directly from Weaviate.
-[Configure a Weaviate collection](#configure-collection) to use a Google AI generative AI model, and Weaviate will perform retrieval augmented generation (RAG) using the specified model and your Google AI API key.
+[Configure a Weaviate collection](#configure-collection) to use a Google generative AI model, and Weaviate will perform retrieval augmented generation (RAG) using the specified model and your Google API key.
-More specifically, Weaviate will perform a search, retrieve the most relevant objects, and then pass them to the Google AI generative model to generate outputs.
+More specifically, Weaviate will perform a search, retrieve the most relevant objects, and then pass them to the Google generative model to generate outputs.
@@ -32,7 +33,7 @@ At the time of writing (November 2023), AI Studio is not available in all region
### Weaviate configuration
-Your Weaviate instance must be configured with the Google AI generative AI integration (`generative-google`) module.
+Your Weaviate instance must be configured with the Google generative AI integration (`generative-google`) module.
:::info Module name change
`generative-google` was called `generative-palm` in Weaviate versions prior to `v1.27`.
@@ -119,7 +120,11 @@ import ApiKeyNote from '../_includes/google-api-key-note.md';
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use a Google AI generative AI model as follows:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use a Google generative AI model as follows:
Note that the required parameters differ between Vertex AI and AI Studio.
@@ -290,7 +295,8 @@ AI Studio:
### Other integrations
-- [Google AI embedding models + Weaviate](./embeddings.md).
+- [Google text embedding models + Weaviate](./embeddings.md).
+- [Google multimodal embedding models + Weaviate](./embeddings-multimodal.md).
### Code examples
diff --git a/developers/weaviate/model-providers/google/index.md b/developers/weaviate/model-providers/google/index.md
index 3188b9de01..93252cea9a 100644
--- a/developers/weaviate/model-providers/google/index.md
+++ b/developers/weaviate/model-providers/google/index.md
@@ -1,5 +1,5 @@
---
-title: Google AI + Weaviate
+title: Google + Weaviate
sidebar_position: 10
image: og/docs/integrations/provider_integrations_google.jpg
# tags: ['model providers', 'google']
@@ -7,41 +7,41 @@ image: og/docs/integrations/provider_integrations_google.jpg
-Google AI offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with [Google AI Studio](https://ai.google.dev/?utm_source=weaviate&utm_medium=referral&utm_campaign=partnerships&utm_content=) and [Google Vertex AI](https://cloud.google.com/vertex-ai) APIs, allowing users to leverage Google AI's models directly within the Weaviate database.
+Google offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with [Google AI Studio](https://ai.google.dev/?utm_source=weaviate&utm_medium=referral&utm_campaign=partnerships&utm_content=) and [Google Vertex AI](https://cloud.google.com/vertex-ai) APIs, allowing users to leverage Google's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
-## Integrations with Google AI
+## Integrations with Google
### Embedding models for semantic search
-Google AI's embedding models transform text data into high-dimensional vector representations, capturing semantic meaning and context.
+Google's embedding models transform text data into high-dimensional vector representations, capturing semantic meaning and context.
-[Weaviate integrates with Google AI's embedding models](./embeddings.md) to enable seamless vectorization of data. This integration allows users to perform semantic and hybrid search operations without the need for additional preprocessing or data transformation steps.
+[Weaviate integrates with Google's embedding models](./embeddings.md) to enable seamless vectorization of data. This integration allows users to perform semantic and hybrid search operations without the need for additional preprocessing or data transformation steps.
-[Google AI embedding integration page](./embeddings.md)
+[Google embedding integration page](./embeddings.md)
### Generative AI models for RAG
-Google AI's generative AI models can generate human-like text based on given prompts and contexts.
+Google's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Google AI's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Google's generative AI models to generate personalized and context-aware responses.
-[Google AI generative AI integration page](./generative.md)
+[Google generative AI integration page](./generative.md)
## Summary
-These integrations enable developers to leverage Google AI's powerful models directly within Weaviate.
+These integrations enable developers to leverage Google's powerful models directly within Weaviate.
In turn, they simplify the process of building AI-driven applications to speed up your development process, so that you can focus on creating innovative solutions.
## Credentials
-You must provide a valid Google AI API credentials to Weaviate for these integrations.
+You must provide a valid Googles API credentials to Weaviate for these integrations.
### Vertex AI
@@ -61,12 +61,6 @@ Go to the relevant integration page to learn how to configure Weaviate with the
- [Multimodal Embeddings](./embeddings-multimodal.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/gpt4all/index.md b/developers/weaviate/model-providers/gpt4all/index.md
index c58846dec5..5fbc0cae2d 100644
--- a/developers/weaviate/model-providers/gpt4all/index.md
+++ b/developers/weaviate/model-providers/gpt4all/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_gpt4all.jpg
-The GPT4All library allows you to easily run a wide range of models on your own device. Weaviate seamlessly integrates with the GPT4All library, allowing users to leverage compatible models directly within the Weaviate database.
+The GPT4All library allows you to easily run a wide range of models on your own device. Weaviate seamlessly integrates with the GPT4All library, allowing users to leverage compatible models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -39,12 +39,6 @@ Go to the relevant integration page to learn how to configure Weaviate with the
- [Text Embeddings](./embeddings.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/huggingface/embeddings.md b/developers/weaviate/model-providers/huggingface/embeddings.md
index de2cb7014a..a339686051 100644
--- a/developers/weaviate/model-providers/huggingface/embeddings.md
+++ b/developers/weaviate/model-providers/huggingface/embeddings.md
@@ -13,8 +13,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with Hugging Face's APIs allows you to access their models' capabilities directly from Weaviate.
@@ -74,6 +76,14 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
## Configure the vectorizer
@@ -99,6 +109,14 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
You must specify one of the [available models](#available-models) for the vectorizer to use.
@@ -116,6 +134,36 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
The following examples show how to configure Hugging Face-specific options.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
#### Model selection parameters
Only select one of the following parameters to specify the model:
@@ -130,6 +178,12 @@ The `passageModel` and `queryModel` parameters are used together to specify a [D
The `endpointURL` parameter is used to specify a [custom Hugging Face Inference Endpoint](https://huggingface.co/inference-endpoints). This parameter overrides the `model`, `passageModel`, and `queryModel` parameters.
:::
+#### Additional parameters
+
+- `options.waitForModel`: If the model is not ready, wait for it rather than returning a `503` error.
+- `options.useGPU`: Use a GPU for inference if your [account plan](https://huggingface.co/inference-api#pricing) supports it.
+- `options.useCache`: Use a cached result if available. (For non-deterministic models to prevent the caching mechanism from being used.)
+
## Data import
After configuring the vectorizer, [import data](../../manage-data/import.mdx) into Weaviate. Weaviate generates embeddings for text objects using the specified model.
@@ -154,6 +208,14 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
:::tip Re-use existing vectors
@@ -192,6 +254,14 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
### Hybrid search
@@ -224,37 +294,18 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
-
-
-## References
-
-#### Other Parameters
-
-- `options.waitForModel`: If the model is not ready, wait for it rather than returning a `503` error.
-- `options.useGPU`: Use a GPU for inference if your [account plan](https://huggingface.co/inference-api#pricing) supports it.
-- `options.useCache`: Use a cached result if available. (For non-deterministic models to prevent the caching mechanism from being used.)
-
-
-
-
-
-
-
+
-
+## References
+
### Available models
You can use any Hugging Face embedding model with `text2vec-huggingface`, including public and private Hugging Face models. [Sentence similarity models](https://huggingface.co/models?pipeline_tag=sentence-similarity&sort=downloads) generally work best.
diff --git a/developers/weaviate/model-providers/huggingface/index.md b/developers/weaviate/model-providers/huggingface/index.md
index a9c220027f..8c420debbd 100644
--- a/developers/weaviate/model-providers/huggingface/index.md
+++ b/developers/weaviate/model-providers/huggingface/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_huggingface.jpg
-Hugging Face offers a wide range of models for natural language processing. Weaviate seamlessly integrates with Hugging Face's Inference API, allowing users to leverage Hugging Face Hub's models directly within the Weaviate database.
+Hugging Face offers a wide range of models for natural language processing. Weaviate seamlessly integrates with Hugging Face's Inference API, allowing users to leverage Hugging Face Hub's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -37,12 +37,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/imagebind/embeddings-multimodal.md b/developers/weaviate/model-providers/imagebind/embeddings-multimodal.md
index 1d32725c87..85bad7badb 100644
--- a/developers/weaviate/model-providers/imagebind/embeddings-multimodal.md
+++ b/developers/weaviate/model-providers/imagebind/embeddings-multimodal.md
@@ -20,7 +20,7 @@ Weaviate's integration with the Meta ImageBind library allows you to access its
[Configure a Weaviate vector index](#configure-the-vectorizer) to use the ImageBind integration, and [configure the Weaviate instance](#weaviate-configuration) with a model image, and Weaviate will generate embeddings for various operations using the specified model in the ImageBind inference container. This feature is called the *vectorizer*.
-At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts queries of one or more modalities into embeddings.
+At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts queries of one or more modalities into embeddings. [Multimodal search operations](#vector-near-media-search) are also supported.
@@ -28,7 +28,7 @@ At [import time](#data-import), Weaviate generates multimodal object embeddings
### Weaviate configuration
-Your Weaviate instance must be configured with the ImageBind vectorizer integration (`multi2vec-bind`) module.
+Your Weaviate instance must be configured with the ImageBind multimodal vectorizer integration (`multi2vec-bind`) module.
For Weaviate Cloud (WCD) users
diff --git a/developers/weaviate/model-providers/imagebind/index.md b/developers/weaviate/model-providers/imagebind/index.md
index e62c2dd29b..b028005a8f 100644
--- a/developers/weaviate/model-providers/imagebind/index.md
+++ b/developers/weaviate/model-providers/imagebind/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_imagebind.jpg
-Meta's ImageBind library can be used with a wide range of models for natural language processing. Weaviate seamlessly integrates with the ImageBind library, allowing users to leverage compatible models directly within the Weaviate database.
+Meta's ImageBind library can be used with a wide range of models for natural language processing. Weaviate seamlessly integrates with the ImageBind library, allowing users to leverage compatible models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -39,12 +39,6 @@ Go to the relevant integration page to learn how to configure Weaviate with the
- [Multimodal Embeddings](./embeddings-multimodal.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/index.md b/developers/weaviate/model-providers/index.md
index fff1243768..8d2f0d0b09 100644
--- a/developers/weaviate/model-providers/index.md
+++ b/developers/weaviate/model-providers/index.md
@@ -20,17 +20,18 @@ This enables an enhanced developed experience, such as the ability to:
| [Anthropic](./anthropic/index.md) | - | [Text](./anthropic/generative.md) | - |
| [Anyscale](./anyscale/index.md) | - | [Text](./anyscale/generative.md) | - |
| [AWS](./aws/index.md) | [Text](./aws/embeddings.md) | [Text](./aws/generative.md) |
-| [Cohere](./cohere/index.md) | [Text](./cohere/embeddings.md) | [Text](./cohere/generative.md) | [Reranker](./cohere/reranker.md) |
+| [Cohere](./cohere/index.md) | [Text](./cohere/embeddings.md), [Multimodal](./cohere/embeddings-multimodal.md) | [Text](./cohere/generative.md) | [Reranker](./cohere/reranker.md) |
| [Databricks](./databricks/index.md) | [Text](./databricks/embeddings.md) | [Text](./databricks/generative.md) | - |
| [FriendliAI](./friendliai/index.md) | - | [Text](./friendliai/generative.md) | - |
| [Google](./google/index.md) | [Text](./google/embeddings.md), [Multimodal](./google/embeddings-multimodal.md) | [Text](./google/generative.md) | - |
| [Hugging Face](./huggingface/index.md) | [Text](./huggingface/embeddings.md) | - | - |
-| [Jina AI](./jinaai/index.md) | [Text](./jinaai/embeddings.md) | - | - |
+| [Jina AI](./jinaai/index.md) | [Text](./jinaai/embeddings.md), [Multimodal](./jinaai/embeddings-multimodal.md) | - | [Reranker](./jinaai/reranker.md) |
| [Mistral](./mistral/index.md) | [Text](./mistral/embeddings.md) | [Text](./mistral/generative.md) | - |
-| [OctoAI](./octoai/index.md) | [Text](./octoai/embeddings.md) | [Text](./octoai/generative.md) | - |
+| [OctoAI (Deprecated)](./octoai/index.md) | [Text](./octoai/embeddings.md) | [Text](./octoai/generative.md) | - |
| [OpenAI](./openai/index.md) | [Text](./openai/embeddings.md) | [Text](./openai/generative.md) | - |
| [Azure OpenAI](./openai-azure/index.md) | [Text](./openai-azure/embeddings.md) | [Text](./openai-azure/generative.md) | - |
-| [Voyage AI](./voyageai/index.md) | [Text](./voyageai/embeddings.md) | - | [Reranker](./voyageai/reranker.md) |
+| [Voyage AI](./voyageai/index.md) | [Text](./voyageai/embeddings.md), [Multimodal](./voyageai/embeddings-multimodal.md) | - | [Reranker](./voyageai/reranker.md) |
+| [Weaviate](./weaviate/index.md) | [Text](./weaviate/embeddings.md) | - | - |
#### Enable all API-based modules
@@ -59,6 +60,56 @@ When a model provider integration for embeddings is enabled, Weaviate automatica
This is done by providing the source data to the integration provider, which then returns the embeddings to Weaviate. The embeddings are then stored in the Weaviate database.
+```mermaid
+%%{init: {
+ 'theme': 'base',
+ 'themeVariables': {
+ 'primaryColor': '#4a5568',
+ 'primaryTextColor': '#2d3748',
+ 'primaryBorderColor': '#718096',
+ 'lineColor': '#718096',
+ 'secondaryColor': '#f7fafc',
+ 'tertiaryColor': '#edf2f7'
+ }
+}}%%
+
+flowchart LR
+ %% Style definitions
+ classDef systemBox fill:#f7fafc,stroke:#3182ce,stroke-width:2px,color:#2d3748,padding:10px
+ classDef weaviateBox fill:#f7fafc,stroke:#2d3748,stroke-width:2px,color:#2d3748,padding:10px
+ classDef providerBox fill:#f7fafc,stroke:#48bb78,stroke-width:2px,color:#2d3748,padding:10px
+ classDef component fill:white,stroke:#718096,stroke-width:1.5px,color:#2d3748,rx:6
+
+ %% Model Provider section (leftmost)
+ subgraph provider["Model Provider"]
+ inference["🤖 Inference API /\nLocal Model"]
+ end
+
+ %% Weaviate section (middle)
+ subgraph weaviate["Weaviate"]
+ vectorizer["🔌 Model Provider\nIntegration"]
+ core["⚡️ Data & vector store"]
+ end
+
+ %% User System (bottom)
+ subgraph user["User System"]
+ data["📄 Data"]
+ end
+
+ %% Connections
+ data -->|"1. Insert objects"| core
+ core -->|"2. Request vector"| vectorizer
+ vectorizer -->|"3. Request vector"| inference
+ inference -->|"4. Vector"| vectorizer
+ vectorizer -->|"5. Vector"| core
+
+ %% Apply styles
+ class user systemBox
+ class weaviate weaviateBox
+ class provider providerBox
+ class data,core,vectorizer,inference component
+```
+
Weaviate generates embeddings for objects as follows:
- Selects properties with `text` or `text[]` data types unless they are configured to be skipped
@@ -72,12 +123,6 @@ If you prefer the text to be lowercased, you can do so by setting the `LOWERCASE
The text is always lowercased for the `text2vec-contextionary` integration.
:::
-## Further resources
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/jinaai/embeddings-multimodal.md b/developers/weaviate/model-providers/jinaai/embeddings-multimodal.md
new file mode 100644
index 0000000000..0261179354
--- /dev/null
+++ b/developers/weaviate/model-providers/jinaai/embeddings-multimodal.md
@@ -0,0 +1,331 @@
+---
+title: Multimodal Embeddings
+sidebar_position: 25
+image: og/docs/integrations/provider_integrations_jinaai.jpg
+# tags: ['model providers', 'jinaai', 'embeddings']
+---
+
+# Jina AI Multimodal Embeddings with Weaviate
+
+:::info Added in `1.25.26`, `1.26.11` and `v1.27.5`
+:::
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
+import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
+import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+
+Weaviate's integration with Jina AI's APIs allows you to access their models' capabilities directly from Weaviate.
+
+[Configure a Weaviate vector index](#configure-the-vectorizer) to use an Jina AI embedding model, and Weaviate will generate embeddings for various operations using the specified model and your Jina AI API key. This feature is called the *vectorizer*.
+
+At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts queries of one or more modalities into embeddings. [Multimodal search operations](#vector-near-media-search) are also supported.
+
+
+
+## Requirements
+
+### Weaviate configuration
+
+Your Weaviate instance must be configured with the Jina AI multimodal vectorizer integration (`multi2vec-jinaai`) module.
+
+
+ For Weaviate Cloud (WCD) users
+
+This integration is enabled by default on Weaviate Cloud (WCD) serverless instances.
+
+
+
+
+ For self-hosted users
+
+- Check the [cluster metadata](../../config-refs/meta.md) to verify if the module is enabled.
+- Follow the [how-to configure modules](../../configuration/modules.md) guide to enable the module in Weaviate.
+
+
+
+### API credentials
+
+You must provide a valid Jina AI API key to Weaviate for this integration. Go to [Jina AI](https://jina.ai/embeddings/) to sign up and obtain an API key.
+
+Provide the API key to Weaviate using one of the following methods:
+
+- Set the `JINAAI_APIKEY` environment variable that is available to Weaviate.
+- Provide the API key at runtime, as shown in the examples below.
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Configure the vectorizer
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a Jina AI embedding model:
+
+
+
+
+
+
+
+
+
+
+
+
+### Select a model
+
+You can specify one of the [available models](#available-models) for the vectorizer to use, as shown in the following configuration example.
+
+
+
+
+
+
+
+
+
+
+
+
+You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
+
+import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
+
+
+ Vectorization behavior
+
+
+
+
+
+### Vectorizer parameters
+
+The following examples show how to configure Jina AI-specific options.
+
+
+
+
+
+
+
+
+
+
+
+
+### Vectorizer parameters
+
+- `model`: The model name.
+- `dimensions`: The number of dimensions for the model.
+ - Note that [not all models](#available-models) support this parameter.
+
+## Data import
+
+After configuring the vectorizer, [import data](../../manage-data/import.mdx) into Weaviate. Weaviate generates embeddings for text objects using the specified model.
+
+
+
+
+
+
+
+
+
+
+
+
+
+:::tip Re-use existing vectors
+If you already have a compatible model vector available, you can provide it directly to Weaviate. This can be useful if you have already generated embeddings using the same model and want to use them in Weaviate, such as when migrating data from another system.
+:::
+
+## Searches
+
+Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified Jina AI model.
+
+
+
+### Vector (near text) search
+
+When you perform a [vector search](../../search/similarity.md#search-with-text), Weaviate converts the text query into an embedding using the specified model and returns the most similar objects from the database.
+
+The query below returns the `n` most similar objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Hybrid search
+
+:::info What is a hybrid search?
+A hybrid search performs a vector search and a keyword (BM25) search, before [combining the results](../../search/hybrid.md#change-the-ranking-method) to return the best matching objects from the database.
+:::
+
+When you perform a [hybrid search](../../search/hybrid.md), Weaviate converts the text query into an embedding using the specified model and returns the best scoring objects from the database.
+
+The query below returns the `n` best scoring objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Vector (near media) search
+
+When you perform a media search such as a [near image search](../../search/similarity.md#search-with-image), Weaviate converts the query into an embedding using the specified model and returns the most similar objects from the database.
+
+To perform a near media search such as near image search, convert the media query into a base64 string and pass it to the search query.
+
+The query below returns the `n` most similar objects to the input image from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+## References
+
+### Available models
+
+- `jina-clip-v2`
+ - This model is a multilingual, multimodal model using [Matryoshka Representation Learning](https://arxiv.org/abs/2205.13147).
+ - It will accept a `dimensions` parameter, which can be any integer between (and including) 64 and 1024. The default value is 1024.
+- `jina-clip-v1`
+ - This model will always return a 768-dimensional embedding.
+
+## Further resources
+
+### Other integrations
+
+- [Jina AI text embedding models + Weaviate](./embeddings.md).
+- [Jina AI reranker models + Weaviate](./reranker.md).
+
+### Code examples
+
+Once the integrations are configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples:
+
+- The [how-to: manage data](../../manage-data/index.md) guides show how to perform data operations (i.e. create, update, delete).
+- The [how-to: search](../../search/index.md) guides show how to perform search operations (i.e. vector, keyword, hybrid) as well as retrieval augmented generation.
+
+### External resources
+
+- Jina AI [Embeddings API documentation](https://jina.ai/embeddings/)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/model-providers/jinaai/embeddings.md b/developers/weaviate/model-providers/jinaai/embeddings.md
index c12cbe7f70..dc7ffb6484 100644
--- a/developers/weaviate/model-providers/jinaai/embeddings.md
+++ b/developers/weaviate/model-providers/jinaai/embeddings.md
@@ -13,8 +13,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with Jina AI's APIs allows you to access their models' capabilities directly from Weaviate.
@@ -74,6 +76,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
## Configure the vectorizer
@@ -99,6 +110,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
### Select a model
@@ -124,6 +144,15 @@ You can specify one of the [available models](#available-models) for the vectori
/>
+
+
+
+
You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
@@ -141,7 +170,7 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
The following examples show how to configure Jina AI-specific options.
-Note that `dimensions` is only used for the `jina-embeddings-v3` model.
+Note that `dimensions` is not applicable for the `jina-embeddings-v2` models.
@@ -162,6 +191,15 @@ Note that `dimensions` is only used for the `jina-embeddings-v3` model.
/>
+
+
+
+
## Data import
@@ -188,6 +226,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -226,6 +273,15 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
### Hybrid search
@@ -258,6 +314,15 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
## References
@@ -270,8 +335,15 @@ The query below returns the `n` best scoring objects from the database, set by `
- `jina-embeddings-v2-base-en` (Default)
- `jina-embeddings-v2-small-en`
+Note that `dimensions` is not applicable for the `jina-embeddings-v2` models.
+
## Further resources
+### Other integrations
+
+- [Jina AI multimodal embedding models + Weaviate](./embeddings-multimodal.md)
+- [Jina AI reranker models + Weaviate](./reranker.md)
+
### Code examples
Once the integrations are configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples:
diff --git a/developers/weaviate/model-providers/jinaai/index.md b/developers/weaviate/model-providers/jinaai/index.md
index 35f2b27f6b..89f9a53fa9 100644
--- a/developers/weaviate/model-providers/jinaai/index.md
+++ b/developers/weaviate/model-providers/jinaai/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_jinaai.jpg
-Jina AI offers a wide range of models for natural language processing. Weaviate seamlessly integrates with Jina AI's APIs, allowing users to leverage Jina AI's models directly within the Weaviate database.
+Jina AI offers a wide range of models for natural language processing. Weaviate seamlessly integrates with Jina AI's APIs, allowing users to leverage Jina AI's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -37,12 +37,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/jinaai/reranker.md b/developers/weaviate/model-providers/jinaai/reranker.md
new file mode 100644
index 0000000000..e007219cac
--- /dev/null
+++ b/developers/weaviate/model-providers/jinaai/reranker.md
@@ -0,0 +1,201 @@
+---
+title: Reranker
+sidebar_position: 70
+image: og/docs/integrations/provider_integrations_jinaai.jpg
+# tags: ['model providers', 'jinaai', 'reranking']
+---
+
+# Jina AI Reranker Models with Weaviate
+
+:::info Added in `v1.26.1`
+:::
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
+import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import PyCode from '!!raw-loader!../_includes/provider.reranker.py';
+import TSCode from '!!raw-loader!../_includes/provider.reranker.ts';
+
+Weaviate's integration with Jina AI's APIs allows you to access their models' capabilities directly from Weaviate.
+
+[Configure a Weaviate collection](#configure-the-reranker) to use a Jina AI reranker model, and Weaviate will use the specified model and your Jina AI API key to rerank search results.
+
+This two-step process involves Weaviate first performing a search and then reranking the results using the specified model.
+
+
+
+## Requirements
+
+### Weaviate configuration
+
+Your Weaviate instance must be configured with the JinaAI reranker integration (`reranker-jinaai`) module.
+
+
+ For Weaviate Cloud (WCD) users
+
+This integration is enabled by default on Weaviate Cloud (WCD) serverless instances.
+
+
+
+
+ For self-hosted users
+
+- Check the [cluster metadata](../../config-refs/meta.md) to verify if the module is enabled.
+- Follow the [how-to configure modules](../../configuration/modules.md) guide to enable the module in Weaviate.
+
+
+
+### API credentials
+
+You must provide a valid JinaAI API key to Weaviate for this integration. Go to [Jina AI](https://jina.ai/embeddings/) to sign up and obtain an API key.
+
+Provide the API key to Weaviate using one of the following methods:
+
+- Set the `JINAAI_APIKEY` environment variable that is available to Weaviate.
+- Provide the API key at runtime, as shown in the examples below.
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Configure the reranker
+
+import MutableRerankerConfig from '/_includes/mutable-reranker-config.md';
+
+
+
+Configure a Weaviate collection to use a Jina AI reranker model as follows:
+
+
+
+
+
+
+
+
+
+
+
+
+### Select a model
+
+You can specify one of the [available models](#available-models) for Weaviate to use, as shown in the following configuration example:
+
+
+
+
+
+
+
+
+
+
+
+
+The [default model](#available-models) is used if no model is specified.
+
+## Reranking query
+
+Once the reranker is configured, Weaviate performs [reranking operations](../../search/rerank.md) using the specified Jina AI model.
+
+More specifically, Weaviate performs an initial search, then reranks the results using the specified model.
+
+Any search in Weaviate can be combined with a reranker to perform reranking operations.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## References
+
+### Available models
+
+- `jina-reranker-v2-base-multilingual` (default)
+- `jina-reranker-v1-base-en`
+- `jina-reranker-v1-turbo-en`
+- `jina-reranker-v1-tiny-en`
+- `jina-colbert-v1-en`
+
+## Further resources
+
+### Other integrations
+
+- [Jina AI text embedding models + Weaviate](./embeddings.md)
+- [Jina AI multimodal embedding models + Weaviate](./embeddings-multimodal.md)
+
+### Code examples
+
+Once the integrations are configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples:
+
+- The [how-to: manage data](../../manage-data/index.md) guides show how to perform data operations (i.e. create, update, delete).
+- The [how-to: search](../../search/index.md) guides show how to perform search operations (i.e. vector, keyword, hybrid) as well as retrieval augmented generation.
+
+### References
+
+- Jina AI [Reranker documentation](https://jina.ai/reranker/)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/model-providers/kubeai/embeddings.md b/developers/weaviate/model-providers/kubeai/embeddings.md
index d355bef69d..1f1a812db4 100644
--- a/developers/weaviate/model-providers/kubeai/embeddings.md
+++ b/developers/weaviate/model-providers/kubeai/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: Implement KubeAI embeddings in Weaviate to enrich data with contextual vectors.
sidebar_position: 20
image: og/docs/integrations/provider_integrations_kubeai.jpg
# tags: ['model providers', 'openai', 'embeddings']
diff --git a/developers/weaviate/model-providers/kubeai/generative.md b/developers/weaviate/model-providers/kubeai/generative.md
index 84e5b95d73..f6688f46ca 100644
--- a/developers/weaviate/model-providers/kubeai/generative.md
+++ b/developers/weaviate/model-providers/kubeai/generative.md
@@ -1,5 +1,6 @@
---
title: Generative AI
+description: Set up generative models with KubeAI in Weaviate for creative data outputs.
sidebar_position: 50
image: og/docs/integrations/provider_integrations_kubeai.jpg
# tags: ['model providers', 'openai', 'generative', 'rag']
diff --git a/developers/weaviate/model-providers/kubeai/index.md b/developers/weaviate/model-providers/kubeai/index.md
index 1e1f44f1ab..c95c84a582 100644
--- a/developers/weaviate/model-providers/kubeai/index.md
+++ b/developers/weaviate/model-providers/kubeai/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_kubeai.jpg
-[KubeAI](https://github.com/substratusai/kubeai) provides offers a wide range of models for natural language processing and generation through OpenAI-style API endpoints. Weaviate seamlessly integrates with KubeAI's APIs, allowing users to leverage any KubeAI models directly within the Weaviate database.
+[KubeAI](https://github.com/substratusai/kubeai) provides offers a wide range of models for natural language processing and generation through OpenAI-style API endpoints. Weaviate seamlessly integrates with KubeAI's APIs, allowing users to leverage any KubeAI models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -29,7 +29,7 @@ KubeAI's embedding models transform text data into high-dimensional vector repre
KubeAI's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with KubeAI's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with KubeAI's generative AI models to generate personalized and context-aware responses.
[KubeAI generative AI integration page](./generative.md)
@@ -48,12 +48,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/mistral/embeddings.md b/developers/weaviate/model-providers/mistral/embeddings.md
index 2cbef69b72..a7e6974271 100644
--- a/developers/weaviate/model-providers/mistral/embeddings.md
+++ b/developers/weaviate/model-providers/mistral/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: Use Mistral embeddings with Weaviate for high-quality vector search results.
sidebar_position: 20
image: og/docs/integrations/provider_integrations_mistral.jpg
# tags: ['model providers', 'mistral', 'embeddings']
@@ -12,8 +13,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with Mistral's APIs allows you to access their models' capabilities directly from Weaviate.
@@ -73,6 +76,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
## Configure the vectorizer
@@ -98,6 +110,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
### Select a model
@@ -123,6 +144,15 @@ You can specify one of the [available models](#available-models) for the vectori
/>
+
+
+
+
The [default model](#available-models) is used if no model is specified.
@@ -160,6 +190,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -198,6 +237,15 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
### Hybrid search
@@ -230,6 +278,15 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
## References
diff --git a/developers/weaviate/model-providers/mistral/generative.md b/developers/weaviate/model-providers/mistral/generative.md
index ba55b33dbb..c28e604063 100644
--- a/developers/weaviate/model-providers/mistral/generative.md
+++ b/developers/weaviate/model-providers/mistral/generative.md
@@ -1,5 +1,6 @@
---
title: Generative AI
+description: "Weaviate's integration with Mistral's APIs allows you to access their models' capabilities directly from Weaviate."
sidebar_position: 50
image: og/docs/integrations/provider_integrations_mistral.jpg
# tags: ['model providers', 'mistral', 'generative', 'rag']
@@ -78,7 +79,11 @@ Provide the API key to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use a Mistral generative model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use a Mistral generative model:
diff --git a/developers/weaviate/model-providers/mistral/index.md b/developers/weaviate/model-providers/mistral/index.md
index 219277d676..4b99b2ac73 100644
--- a/developers/weaviate/model-providers/mistral/index.md
+++ b/developers/weaviate/model-providers/mistral/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_mistral.jpg
-Mistral offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Mistral's APIs, allowing users to leverage Mistral's models directly within the Weaviate database.
+Mistral offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with Mistral's APIs, allowing users to leverage Mistral's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -29,7 +29,7 @@ Mistral's embedding models transform text data into high-dimensional vector repr
Mistral's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Mistral's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Mistral's generative AI models to generate personalized and context-aware responses.
[Mistral generative AI integration page](./generative.md)
@@ -48,12 +48,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/octoai/_category_.json b/developers/weaviate/model-providers/octoai/_category_.json
index d812db4d45..42d4b1b3a7 100644
--- a/developers/weaviate/model-providers/octoai/_category_.json
+++ b/developers/weaviate/model-providers/octoai/_category_.json
@@ -1,4 +1,4 @@
{
- "label": "OctoAI",
+ "label": "OctoAI (Deprecated)",
"position": 247
}
diff --git a/developers/weaviate/model-providers/octoai/_includes/octoai_deprecation.md b/developers/weaviate/model-providers/octoai/_includes/octoai_deprecation.md
new file mode 100644
index 0000000000..8a2f235a7f
--- /dev/null
+++ b/developers/weaviate/model-providers/octoai/_includes/octoai_deprecation.md
@@ -0,0 +1,70 @@
+:::warning Deprecated integrations
+
+### OctoAI integrations are deprecated
+
+
+
+OctoAI announced that they are winding down the commercial availability of its services by **31 October 2024**. Accordingly, the Weaviate OctoAI integrations are deprecated. Do not use these integrations for new projects.
+
+
+If you have a collection that is using an OctoAI integration, consider your options depending on whether you are using OctoAI's embedding models ([your options](#for-collections-with-octoai-embedding-integrations)) or generative models ([your options](#for-collections-with-octoai-generative-ai-integrations)).
+
+#### For collections with OctoAI embedding integrations
+
+OctoAI provided `thenlper/gte-large` as the embedding model. This model is also available through the [Hugging Face API](../../huggingface/embeddings.md).
+
+
+
+After the shutdown date, this model will no longer be available through OctoAI. If you are using this integration, you have the following options:
+
+
+**Option 1: Use the existing collection, and provide your own vectors**
+
+
+You can continue to use the existing collection, provided that you rely on some other method to generate the required embeddings yourself for any new data, and for queries. If you are unfamiliar with the "bring your own vectors" approach, [refer to this starter guide](../../../starter-guides/custom-vectors.mdx).
+
+
+**Option 2: Migrate to a new collection with another model provider**
+
+Alternatively, you can migrate your data to a new collection ([read how](#how-to-migrate)). At this point, you can re-use the existing embeddings or choose a new model.
+
+
+- **Re-using the existing embeddings** will save on time and inference costs.
+- **Choosing a new model** will allow you to explore new models and potentially improve the performance of your application.
+
+If you would like to re-use the existing embeddings, you must select a model provider (e.g. [Hugging Face API](../../huggingface/embeddings.md)) that offers the same embedding model.
+
+
+You can also select a new model with any embedding model provider. This will require you to re-generate the embeddings for your data, as the existing embeddings will not be compatible with the new model.
+
+
+#### For collections with OctoAI generative AI integrations
+
+If you are only using the generative AI integration, you do not need to migrate your data to a new collection.
+
+
+Follow [this how-to](/developers/weaviate/manage-data/collections#update-the-generative-model-integration) to re-configure your collection with a new generative AI model provider. Note this requires Weaviate `v1.25.23`, `v1.26.8`, `v1.27.1`, or later.
+
+
+You can select any model provider that offers generative AI models.
+
+
+If you would like to continue to use the same model that you used with OctoAI, providers such as [Anyscale](../../anyscale/generative.md), [FriendliAI](../../friendliai/generative.md), [Mistral](../../mistral/generative.md) or local models with [Ollama](../../ollama/generative.md) each offer some of the suite of models that OctoAI provided.
+
+
+#### How to migrate
+
+An outline of the migration process is as follows:
+
+
+- Create a new collection with the desired model provider integration(s).
+- Export the data from the existing collection.
+ - (Optional) To re-use the existing embeddings, export the data with the existing embeddings.
+- Import the data into the new collection.
+ - (Optional) To re-use the existing embeddings, import the data with the existing embeddings.
+- Update your application to use the new collection.
+
+
+See [How-to manage data: migrate data](../../../manage-data/migrate.mdx) for examples on migrating data objects between collections.
+
+:::
diff --git a/developers/weaviate/model-providers/octoai/embeddings.md b/developers/weaviate/model-providers/octoai/embeddings.md
index 62ec0f66a7..24a88833f7 100644
--- a/developers/weaviate/model-providers/octoai/embeddings.md
+++ b/developers/weaviate/model-providers/octoai/embeddings.md
@@ -1,12 +1,15 @@
---
-title: Text Embeddings
+title: Text Embeddings (Deprecated)
sidebar_position: 20
image: og/docs/integrations/provider_integrations_octoai.jpg
# tags: ['model providers', 'octoai', 'embeddings']
---
-# OctoAI Embeddings with Weaviate
+import OctoAIDeprecationNote from './_includes/octoai_deprecation.md';
+
+
+# OctoAI Embeddings with Weaviate
:::info Added in `v1.25.0`
:::
diff --git a/developers/weaviate/model-providers/octoai/generative.md b/developers/weaviate/model-providers/octoai/generative.md
index aa6a0398b4..8da577a935 100644
--- a/developers/weaviate/model-providers/octoai/generative.md
+++ b/developers/weaviate/model-providers/octoai/generative.md
@@ -1,10 +1,14 @@
---
-title: Generative AI
+title: Generative AI (Deprecated)
sidebar_position: 50
image: og/docs/integrations/provider_integrations_octoai.jpg
# tags: ['model providers', 'octoai', 'generative', 'rag']
---
+import OctoAIDeprecationNote from './_includes/octoai_deprecation.md';
+
+
+
# OctoAI Generative AI with Weaviate
@@ -81,7 +85,11 @@ Provide the API key to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use an OctoAI generative AI model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use an OctoAI generative AI model:
diff --git a/developers/weaviate/model-providers/octoai/index.md b/developers/weaviate/model-providers/octoai/index.md
index d73eea526b..e58e7574e2 100644
--- a/developers/weaviate/model-providers/octoai/index.md
+++ b/developers/weaviate/model-providers/octoai/index.md
@@ -1,5 +1,5 @@
---
-title: OctoAI + Weaviate
+title: OctoAI + Weaviate (Deprecated)
sidebar_position: 10
image: og/docs/integrations/provider_integrations_octoai.jpg
# tags: ['model providers', 'octoai']
@@ -7,10 +7,14 @@ image: og/docs/integrations/provider_integrations_octoai.jpg
+import OctoAIDeprecationNote from './_includes/octoai_deprecation.md';
+
+
+
:::info Added in `v1.25.0`
:::
-OctoAI offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with OctoAI's APIs, allowing users to leverage OctoAI's models directly within the Weaviate database.
+OctoAI offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with OctoAI's APIs, allowing users to leverage OctoAI's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -32,7 +36,7 @@ OctoAI's embedding models transform text data into high-dimensional vector repre
OctoAI's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with OctoAI's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with OctoAI's generative AI models to generate personalized and context-aware responses.
[OctoAI generative AI integration page](./generative.md)
@@ -51,12 +55,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/ollama/embeddings.md b/developers/weaviate/model-providers/ollama/embeddings.md
index eb26b41b5d..816afe0aac 100644
--- a/developers/weaviate/model-providers/ollama/embeddings.md
+++ b/developers/weaviate/model-providers/ollama/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: Ollama Embedding Provider
sidebar_position: 20
image: og/docs/integrations/provider_integrations_ollama.jpg
# tags: ['model providers', 'ollama', 'embeddings']
diff --git a/developers/weaviate/model-providers/ollama/generative.md b/developers/weaviate/model-providers/ollama/generative.md
index 902ae9deaa..4eae5cf366 100644
--- a/developers/weaviate/model-providers/ollama/generative.md
+++ b/developers/weaviate/model-providers/ollama/generative.md
@@ -1,5 +1,6 @@
---
title: Generative AI
+description: Ollama Generative Model Provider
sidebar_position: 50
image: og/docs/integrations/provider_integrations_ollama.jpg
# tags: ['model providers', 'ollama', 'generative', 'rag']
@@ -79,7 +80,11 @@ As this integration connects to a local Ollama container, no additional credenti
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use an Ollama generative model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use an Ollama generative model:
diff --git a/developers/weaviate/model-providers/ollama/index.md b/developers/weaviate/model-providers/ollama/index.md
index 9865ba0461..2aa4019ec0 100644
--- a/developers/weaviate/model-providers/ollama/index.md
+++ b/developers/weaviate/model-providers/ollama/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_ollama.jpg
-The Ollama library allows you to easily run a wide range of models on your own device. Weaviate seamlessly integrates with the Ollama library, allowing users to leverage compatible models directly within the Weaviate database.
+The Ollama library allows you to easily run a wide range of models on your own device. Weaviate seamlessly integrates with the Ollama library, allowing users to leverage compatible models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -31,7 +31,7 @@ Ollama's embedding models transform text data into high-dimensional vector repre
Ollama's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Ollama's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Ollama's generative AI models to generate personalized and context-aware responses.
[Ollama generative AI integration page](./generative.md)
@@ -50,12 +50,6 @@ Go to the relevant integration page to learn how to configure Weaviate with the
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/openai-azure/embeddings.md b/developers/weaviate/model-providers/openai-azure/embeddings.md
index e8ed75e295..5c96019aeb 100644
--- a/developers/weaviate/model-providers/openai-azure/embeddings.md
+++ b/developers/weaviate/model-providers/openai-azure/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: "Weaviate's integration with Azure OpenAI's APIs allows you to access their models' capabilities directly from Weaviate."
sidebar_position: 20
image: og/docs/integrations/provider_integrations_openai_azure.jpg
# tags: ['model providers', 'azure', 'openai', 'embeddings']
@@ -13,8 +14,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with Azure OpenAI's APIs allows you to access their models' capabilities directly from Weaviate.
@@ -74,6 +77,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
## Configure the vectorizer
@@ -101,6 +113,15 @@ To select the model, specify the Azure resource name.
/>
+
+
+
+
import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
@@ -139,6 +160,15 @@ The following examples show how to configure Azure OpenAI-specific options.
/>
+
+
+
+
For further details on these parameters, see consult the [Azure OpenAI API documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/).
@@ -167,6 +197,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -205,6 +244,15 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
### Hybrid search
@@ -237,6 +285,15 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
## References
diff --git a/developers/weaviate/model-providers/openai-azure/generative.md b/developers/weaviate/model-providers/openai-azure/generative.md
index 271961fb96..148b561777 100644
--- a/developers/weaviate/model-providers/openai-azure/generative.md
+++ b/developers/weaviate/model-providers/openai-azure/generative.md
@@ -1,5 +1,6 @@
---
title: Generative AI
+description: OpenAI-Azure Generative Model Provider
sidebar_position: 50
image: og/docs/integrations/provider_integrations_openai_azure.jpg
# tags: ['model providers', 'azure', 'openai', 'generative', 'rag']
@@ -78,7 +79,11 @@ Provide the API key to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use an OpenAI Azure generative model.
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use an OpenAI Azure generative model.
To select the model, specify the Azure resource name.
diff --git a/developers/weaviate/model-providers/openai-azure/index.md b/developers/weaviate/model-providers/openai-azure/index.md
index 8e4d3c97f3..d238251c54 100644
--- a/developers/weaviate/model-providers/openai-azure/index.md
+++ b/developers/weaviate/model-providers/openai-azure/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_openai_azure.jpg
-Microsoft Azure offers a wide range of OpenAI models for natural language processing and generation. Weaviate seamlessly integrates with Microsoft Azure's APIs, allowing users to leverage OpenAI's models directly within the Weaviate database.
+Microsoft Azure offers a wide range of OpenAI models for natural language processing and generation. Weaviate seamlessly integrates with Microsoft Azure's APIs, allowing users to leverage OpenAI's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -29,7 +29,7 @@ Azure OpenAI's embedding models transform text data into high-dimensional vector
Azure OpenAI's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Azure OpenAI's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with Azure OpenAI's generative AI models to generate personalized and context-aware responses.
[Azure OpenAI generative AI integration page](./generative.md)
@@ -48,12 +48,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/openai/embeddings.md b/developers/weaviate/model-providers/openai/embeddings.md
index 468eb626d5..5012adfb53 100644
--- a/developers/weaviate/model-providers/openai/embeddings.md
+++ b/developers/weaviate/model-providers/openai/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: OpenAI Embedding Model Provider
sidebar_position: 20
image: og/docs/integrations/provider_integrations_openai.jpg
# tags: ['model providers', 'openai', 'embeddings']
@@ -16,8 +17,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
Weaviate's integration with OpenAI's APIs allows you to access their models' capabilities directly from Weaviate.
@@ -79,6 +82,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
## Configure the vectorizer
@@ -104,6 +116,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
### Select a model
@@ -133,6 +154,15 @@ For `v3` models such as `text-embedding-3-large`, provide the model name and opt
/>
+
+
+
+
#### For older model families (e.g. `ada`)
@@ -158,6 +188,15 @@ For older models such as `text-embedding-ada-002`, provide the model name (`ada`
/>
+
+
+
+
You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
@@ -208,6 +247,15 @@ The following examples show how to configure OpenAI-specific options.
/>
+
+
+
+
For further details on model parameters, see the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/embeddings).
@@ -236,6 +284,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -274,6 +331,15 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
### Hybrid search
@@ -306,6 +372,15 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
## References
diff --git a/developers/weaviate/model-providers/openai/generative.md b/developers/weaviate/model-providers/openai/generative.md
index ffc589f1c8..7383f7e87b 100644
--- a/developers/weaviate/model-providers/openai/generative.md
+++ b/developers/weaviate/model-providers/openai/generative.md
@@ -1,5 +1,6 @@
---
title: Generative AI
+description: OpenAI Generative Model Provider
sidebar_position: 50
image: og/docs/integrations/provider_integrations_openai.jpg
# tags: ['model providers', 'openai', 'generative', 'rag']
@@ -83,7 +84,11 @@ Provide the API key to Weaviate using one of the following methods:
## Configure collection
-[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-module) as follows to use an OpenAI generative AI model:
+import MutableGenerativeConfig from '/_includes/mutable-generative-config.md';
+
+
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-generative-model-integration) as follows to use an OpenAI generative AI model:
@@ -236,6 +241,8 @@ In other words, when you have `n` search results, the generative model generates
* [gpt-4](https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo)
* [gpt-4-1106-preview](https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo)
* [gpt-4-32k](https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo)
+* [gpt-4o](https://platform.openai.com/docs/models#gpt-4o)
+* [gpt-4o-mini](https://platform.openai.com/docs/models#gpt-4o-mini) (Added in v1.26.7)
Older models
diff --git a/developers/weaviate/model-providers/openai/index.md b/developers/weaviate/model-providers/openai/index.md
index 4c5376b014..c392ec5ab5 100644
--- a/developers/weaviate/model-providers/openai/index.md
+++ b/developers/weaviate/model-providers/openai/index.md
@@ -1,5 +1,6 @@
---
title: OpenAI + Weaviate
+description: "OpenAI offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with OpenAI's APIs, allowing users to leverage OpenAI's models directly from the Weaviate database."
sidebar_position: 10
image: og/docs/integrations/provider_integrations_openai.jpg
# tags: ['model providers', 'openai']
@@ -11,7 +12,7 @@ image: og/docs/integrations/provider_integrations_openai.jpg
For Azure OpenAI integration docs, see [this page instead](../openai-azure/index.md).
:::
-OpenAI offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with OpenAI's APIs, allowing users to leverage OpenAI's models directly within the Weaviate database.
+OpenAI offers a wide range of models for natural language processing and generation. Weaviate seamlessly integrates with OpenAI's APIs, allowing users to leverage OpenAI's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -33,7 +34,7 @@ OpenAI's embedding models transform text data into high-dimensional vector repre
OpenAI's generative AI models can generate human-like text based on given prompts and contexts.
-[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly within the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with OpenAI's generative AI models to generate personalized and context-aware responses.
+[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate database. This combines Weaviate's efficient storage and fast retrieval capabilities with OpenAI's generative AI models to generate personalized and context-aware responses.
[OpenAI generative AI integration page](./generative.md)
@@ -52,12 +53,6 @@ Then, go to the relevant integration page to learn how to configure Weaviate wit
- [Text Embeddings](./embeddings.md)
- [Generative AI](./generative.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/transformers/embeddings-custom-image.md b/developers/weaviate/model-providers/transformers/embeddings-custom-image.md
index 6e0b5aa541..beab99fab5 100644
--- a/developers/weaviate/model-providers/transformers/embeddings-custom-image.md
+++ b/developers/weaviate/model-providers/transformers/embeddings-custom-image.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings (custom)
+description: Transformers Custom Image Embeddings
sidebar_position: 25
image: og/docs/integrations/provider_integrations_transformers.jpg
# tags: ['model providers', 'huggingface', 'embeddings', 'transformers']
diff --git a/developers/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md b/developers/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md
index 6bee817bdd..ec4dfab29e 100644
--- a/developers/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md
+++ b/developers/weaviate/model-providers/transformers/embeddings-multimodal-custom-image.md
@@ -1,5 +1,6 @@
---
title: Multimodal Embeddings (custom)
+description: Transformers Custom Image Multimodal Embeddings
sidebar_position: 35
image: og/docs/integrations/provider_integrations_transformers.jpg
# tags: ['model providers', 'huggingface', 'embeddings', 'clip']
diff --git a/developers/weaviate/model-providers/transformers/embeddings-multimodal.md b/developers/weaviate/model-providers/transformers/embeddings-multimodal.md
index 557ce03bb9..2800b378f9 100644
--- a/developers/weaviate/model-providers/transformers/embeddings-multimodal.md
+++ b/developers/weaviate/model-providers/transformers/embeddings-multimodal.md
@@ -1,5 +1,6 @@
---
title: Multimodal (CLIP) Embeddings
+description: Transformers Multimodal Embeddings
sidebar_position: 30
image: og/docs/integrations/provider_integrations_transformers.jpg
# tags: ['model providers', 'transformers', 'embeddings']
@@ -20,7 +21,7 @@ Weaviate's integration with the Hugging Face Transformers library allows you to
[Configure a Weaviate vector index](#configure-the-vectorizer) to use the CLIP integration, and [configure the Weaviate instance](#weaviate-configuration) with a model image, and Weaviate will generate embeddings for various operations using the specified model in the CLIP inference container. This feature is called the *vectorizer*.
-At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts queries of one or more modalities into embeddings.
+At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts queries of one or more modalities into embeddings. [Multimodal search operations](#vector-near-media-search) are also supported.
@@ -28,7 +29,7 @@ At [import time](#data-import), Weaviate generates multimodal object embeddings
### Weaviate configuration
-Your Weaviate instance must be configured with the CLIP vectorizer integration (`multi2vec-clip`) module.
+Your Weaviate instance must be configured with the CLIP multimodal vectorizer integration (`multi2vec-clip`) module.
For Weaviate Cloud (WCD) users
diff --git a/developers/weaviate/model-providers/transformers/embeddings.md b/developers/weaviate/model-providers/transformers/embeddings.md
index 4060176742..1823513c62 100644
--- a/developers/weaviate/model-providers/transformers/embeddings.md
+++ b/developers/weaviate/model-providers/transformers/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: "Weaviate's integration with the Hugging Face Transformers library allows you to access their models' capabilities directly from Weaviate."
sidebar_position: 20
image: og/docs/integrations/provider_integrations_transformers.jpg
# tags: ['model providers', 'huggingface', 'embeddings', 'transformers']
diff --git a/developers/weaviate/model-providers/transformers/index.md b/developers/weaviate/model-providers/transformers/index.md
index 6ae4bd2c67..bfdcd22020 100644
--- a/developers/weaviate/model-providers/transformers/index.md
+++ b/developers/weaviate/model-providers/transformers/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_transformers.jpg
-Hugging Face's Transformers library can be used with a wide range of models for natural language processing. Weaviate seamlessly integrates with the Transformers library, allowing users to leverage compatible models directly within the Weaviate database.
+Hugging Face's Transformers library can be used with a wide range of models for natural language processing. Weaviate seamlessly integrates with the Transformers library, allowing users to leverage compatible models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -42,12 +42,6 @@ Go to the relevant integration page to learn how to configure Weaviate with the
- [Multimodal Embeddings](./embeddings-multimodal.md)
- [Multimodal Embeddings (custom image)](./embeddings-multimodal-custom-image.md)
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
-
## Questions and feedback
import DocsFeedback from '/_includes/docs-feedback.mdx';
diff --git a/developers/weaviate/model-providers/transformers/reranker.md b/developers/weaviate/model-providers/transformers/reranker.md
index fbedb1bb6f..dc57ce4609 100644
--- a/developers/weaviate/model-providers/transformers/reranker.md
+++ b/developers/weaviate/model-providers/transformers/reranker.md
@@ -1,5 +1,6 @@
---
title: Reranker
+description: Enhance search results with the transformers reranker module in Weaviate.
sidebar_position: 80
image: og/docs/integrations/provider_integrations_transformers.jpg
# tags: ['model providers', 'transformers', 'reranking']
@@ -132,6 +133,10 @@ As this integration runs a local container with the transformers model, no addit
## Configure the reranker
+import MutableRerankerConfig from '/_includes/mutable-reranker-config.md';
+
+
+
Configure a Weaviate collection to use a Transformer reranker model as follows:
diff --git a/developers/weaviate/model-providers/voyageai/embeddings-multimodal.md b/developers/weaviate/model-providers/voyageai/embeddings-multimodal.md
new file mode 100644
index 0000000000..e5f0a58d15
--- /dev/null
+++ b/developers/weaviate/model-providers/voyageai/embeddings-multimodal.md
@@ -0,0 +1,326 @@
+---
+title: Multimodal Embeddings
+description: "Weaviate's integration with VoyageAI's APIs allows you to access their models' capabilities directly from Weaviate."
+sidebar_position: 25
+image: og/docs/integrations/provider_integrations_voyageai.jpg
+# tags: ['model providers', 'voyageai', 'embeddings']
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
+import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
+import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+
+# VoyageAI Multimodal Embeddings with Weaviate
+
+:::info Added in `v1.25.28`, `v1.26.12`, `1.27.8`
+:::
+
+Weaviate's integration with VoyageAI's APIs allows you to access their models' capabilities directly from Weaviate.
+
+[Configure a Weaviate vector index](#configure-the-vectorizer) to use a VoyageAI embedding model, and Weaviate will generate embeddings for various operations using the specified model and your VoyageAI API key. This feature is called the *vectorizer*.
+
+Note this integration does not support Voyage AI's "interleaving" input mode. For this type of usage, import your own vectors with objects as [shown in this guide](../../starter-guides/custom-vectors.mdx).
+
+At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts text queries into embeddings. [Multimodal search operations](#vector-near-media-search) are also supported.
+
+
+
+## Requirements
+
+### Weaviate configuration
+
+Your Weaviate instance must be configured with the VoyageAI vectorizer integration (`multi2vec-voyageai`) module.
+
+
+ For Weaviate Cloud (WCD) users
+
+This integration is enabled by default on Weaviate Cloud (WCD) serverless instances.
+
+
+
+
+ For self-hosted users
+
+- Check the [cluster metadata](../../config-refs/meta.md) to verify if the module is enabled.
+- Follow the [how-to configure modules](../../configuration/modules.md) guide to enable the module in Weaviate.
+
+
+
+### API credentials
+
+You must provide a valid VoyageAI API key to Weaviate for this integration. Go to [VoyageAI](https://voyageai.com/) to sign up and obtain an API key.
+
+Provide the API key to Weaviate using one of the following methods:
+
+- Set the `VOYAGEAI_APIKEY` environment variable that is available to Weaviate.
+- Provide the API key at runtime, as shown in the examples below.
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Configure the vectorizer
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a VoyageAI embedding model:
+
+
+
+
+
+
+
+
+
+
+
+
+### Select a model
+
+You can specify one of the [available models](#available-models) for the vectorizer to use, as shown in the following configuration example.
+
+
+
+
+
+
+
+
+
+
+
+
+You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
+
+import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
+
+
+ Vectorization behavior
+
+
+
+
+
+### Vectorizer parameters
+
+The following examples show how to configure VoyageAI-specific options.
+
+
+
+
+
+
+
+
+
+
+
+
+For further details on model parameters, see the [VoyageAI API documentation](https://docs.voyageai.com/reference/multimodal-embeddings-api).
+
+## Data import
+
+After configuring the vectorizer, [import data](../../manage-data/import.mdx) into Weaviate. Weaviate generates embeddings for text objects using the specified model.
+
+
+
+
+
+
+
+
+
+
+
+
+
+:::tip Re-use existing vectors
+If you already have a compatible model vector available, you can provide it directly to Weaviate. This can be useful if you have already generated embeddings using the same model and want to use them in Weaviate, such as when migrating data from another system.
+:::
+
+## Searches
+
+Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified VoyageAI model.
+
+
+
+### Vector (near text) search
+
+When you perform a [vector search](../../search/similarity.md#search-with-text), Weaviate converts the text query into an embedding using the specified model and returns the most similar objects from the database.
+
+The query below returns the `n` most similar objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Hybrid search
+
+:::info What is a hybrid search?
+A hybrid search performs a vector search and a keyword (BM25) search, before [combining the results](../../search/hybrid.md#change-the-ranking-method) to return the best matching objects from the database.
+:::
+
+When you perform a [hybrid search](../../search/hybrid.md), Weaviate converts the text query into an embedding using the specified model and returns the best scoring objects from the database.
+
+The query below returns the `n` best scoring objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Vector (near media) search
+
+When you perform a media search such as a [near image search](../../search/similarity.md#search-with-image), Weaviate converts the query into an embedding using the specified model and returns the most similar objects from the database.
+
+To perform a near media search such as near image search, convert the media query into a base64 string and pass it to the search query.
+
+The query below returns the `n` most similar objects to the input image from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+## References
+
+### Available models
+
+- `voyage-multimodal-3` (Default)
+
+## Further resources
+
+### Other integrations
+
+- [VoyageAI text embedding models + Weaviate](./embeddings.md).
+- [VoyageAI reranker models + Weaviate](./reranker.md).
+
+### Code examples
+
+Once the integrations are configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples:
+
+- The [how-to: manage data](../../manage-data/index.md) guides show how to perform data operations (i.e. create, update, delete).
+- The [how-to: search](../../search/index.md) guides show how to perform search operations (i.e. vector, keyword, hybrid) as well as retrieval augmented generation.
+
+### External resources
+
+- VoyageAI [Multimodal Embed API documentation](https://docs.voyageai.com/reference/multimodal-embeddings-api)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/model-providers/voyageai/embeddings.md b/developers/weaviate/model-providers/voyageai/embeddings.md
index 02fe0b215b..e5640e6e6f 100644
--- a/developers/weaviate/model-providers/voyageai/embeddings.md
+++ b/developers/weaviate/model-providers/voyageai/embeddings.md
@@ -1,5 +1,6 @@
---
title: Text Embeddings
+description: "Weaviate's integration with Voyage AI's APIs allows you to access their models' capabilities directly from Weaviate."
sidebar_position: 20
image: og/docs/integrations/provider_integrations_voyageai.jpg
# tags: ['model providers', 'voyageai', 'embeddings']
@@ -11,8 +12,10 @@ import TabItem from '@theme/TabItem';
import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
import PyConnect from '!!raw-loader!../_includes/provider.connect.py';
import TSConnect from '!!raw-loader!../_includes/provider.connect.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect/main.go';
import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
# Voyage AI Embeddings with Weaviate
@@ -74,6 +77,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
## Configure the vectorizer
@@ -99,6 +111,15 @@ Provide the API key to Weaviate using one of the following methods:
/>
+
+
+
+
### Select a model
@@ -124,6 +145,15 @@ You can specify one of the [available models](#available-models) for the vectori
/>
+
+
+
+
You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
@@ -160,6 +190,15 @@ The following examples show how to configure Voyage AI-specific options.
/>
+
+
+
+
For further details on model parameters, see the [Voyage AI Embedding API documentation](https://docs.voyageai.com/docs/embeddings).
@@ -188,6 +227,15 @@ After configuring the vectorizer, [import data](../../manage-data/import.mdx) in
/>
+
+
+
+
:::tip Re-use existing vectors
@@ -226,6 +274,15 @@ The query below returns the `n` most similar objects from the database, set by `
/>
+
+
+
+
### Hybrid search
@@ -258,6 +315,15 @@ The query below returns the `n` best scoring objects from the database, set by `
/>
+
+
+
+
## References
@@ -266,7 +332,7 @@ The query below returns the `n` best scoring objects from the database, set by `
- voyage-3 (default)
- voyage-3-lite
-- voyage-large-2 (default up to `v1.24.24`, `v1.25.17`, `v1.26.4`)
+- voyage-large-2 (default for <= `v1.24.24`, `v1.25.17`, `v1.26.4`)
- voyage-code-2
- voyage-2
- voyage-law-2
@@ -296,6 +362,7 @@ The query below returns the `n` best scoring objects from the database, set by `
### Other integrations
+- [Voyage AI multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md)
- [Voyage AI reranker models + Weaviate](./embeddings.md).
### Code examples
diff --git a/developers/weaviate/model-providers/voyageai/index.md b/developers/weaviate/model-providers/voyageai/index.md
index b7852ce2ea..aff6476ffc 100644
--- a/developers/weaviate/model-providers/voyageai/index.md
+++ b/developers/weaviate/model-providers/voyageai/index.md
@@ -7,7 +7,7 @@ image: og/docs/integrations/provider_integrations_voyageai.jpg
-Voyage AI offers a wide range of models for natural language processing. Weaviate seamlessly integrates with Voyage AI's APIs, allowing users to leverage Voyage AI's models directly within the Weaviate database.
+Voyage AI offers a wide range of models for natural language processing. Weaviate seamlessly integrates with Voyage AI's APIs, allowing users to leverage Voyage AI's models directly from the Weaviate database.
These integrations empower developers to build sophisticated AI-driven applications with ease.
@@ -19,9 +19,10 @@ These integrations empower developers to build sophisticated AI-driven applicati
Voyage AI's embedding models transform text data into high-dimensional vector representations, capturing semantic meaning and context.
-[Weaviate integrates with Voyage AI's embedding models](./embeddings.md) to enable seamless vectorization of data. This integration allows users to perform semantic and hybrid search operations without the need for additional preprocessing or data transformation steps.
+[Weaviate integrates with Voyage AI's embedding models](./embeddings.md) and [multimodal embedding models](./embeddings-multimodal.md) to enable seamless vectorization of data. This integration allows users to perform semantic and hybrid search operations without the need for additional preprocessing or data transformation steps.
-[Voyage AI embedding integration page](./embeddings.md)
+- [Voyage AI embedding integration page](./embeddings.md)
+- [Voyage AI multimodal embedding integration page](./embeddings-multimodal.md)
### Reranker models
@@ -46,12 +47,8 @@ You must provide a valid Voyage AI API key to Weaviate for these integrations. G
Then, go to the relevant integration page to learn how to configure Weaviate with the Voyage AI models and start using them in your applications.
- [Text Embeddings](./embeddings.md)
-
-## Other third party integrations
-
-import IntegrationLinkBack from '/_includes/integrations/link-back.mdx';
-
-
+- [Multimodal Embeddings](./embeddings-multimodal.md)
+- [Rerankers](./reranker.md)
## Questions and feedback
diff --git a/developers/weaviate/model-providers/voyageai/reranker.md b/developers/weaviate/model-providers/voyageai/reranker.md
index 228e1c0982..a19308770a 100644
--- a/developers/weaviate/model-providers/voyageai/reranker.md
+++ b/developers/weaviate/model-providers/voyageai/reranker.md
@@ -1,5 +1,6 @@
---
title: Reranker
+description: VoyageAI Reranker Model Provider
sidebar_position: 70
image: og/docs/integrations/provider_integrations_voyageai.jpg
# tags: ['model providers', 'voyageai', 'reranking']
@@ -78,6 +79,10 @@ Provide the API key to Weaviate using one of the following methods:
## Configure the reranker
+import MutableRerankerConfig from '/_includes/mutable-reranker-config.md';
+
+
+
Configure a Weaviate collection to use a Voyage AI reranker model as follows:
@@ -165,6 +170,7 @@ Any search in Weaviate can be combined with a reranker to perform reranking oper
### Other integrations
- [Voyage AI embedding models + Weaviate](./embeddings.md).
+- [Voyage AI multimodal embedding embeddings models + Weaviate](./embeddings-multimodal.md)
### Code examples
diff --git a/developers/weaviate/model-providers/weaviate/_category_.json b/developers/weaviate/model-providers/weaviate/_category_.json
new file mode 100644
index 0000000000..5bb52a5b11
--- /dev/null
+++ b/developers/weaviate/model-providers/weaviate/_category_.json
@@ -0,0 +1,4 @@
+{
+ "label": "Weaviate",
+ "position": 280
+}
diff --git a/developers/weaviate/model-providers/weaviate/embeddings.md b/developers/weaviate/model-providers/weaviate/embeddings.md
new file mode 100644
index 0000000000..739dd7cbd0
--- /dev/null
+++ b/developers/weaviate/model-providers/weaviate/embeddings.md
@@ -0,0 +1,424 @@
+---
+title: Text Embeddings
+description: "Weaviate Embeddings' models can be accessed directly from a Weaviate Cloud instance."
+sidebar_position: 20
+image: og/docs/integrations/provider_integrations_wes.jpg
+# tags: ['model providers', 'weaviate', 'wes', 'weaviate embeddings']
+---
+
+:::info Access to Weaviate Embeddings during technical preview
+Weaviate Embeddings is currently in free technical preview.
+
+To try out Weaviate Embeddings at this time, please [contact us](https://events.weaviate.io/embeddings-preview) to request access.
+:::
+
+# Weaviate Embeddings
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock';
+import PyConnect from '!!raw-loader!../_includes/provider.connect.weaviate.py';
+import TSConnect from '!!raw-loader!../_includes/provider.connect.weaviate.ts';
+import GoConnect from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/1-connect-weaviate-embeddings/main.go';
+import JavaConnect from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ConnectWeaviateEmbeddings.java';
+import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py';
+import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts';
+import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/model-providers/2-usage-text/main.go';
+import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/UsageWeaviateTextEmbeddings.java';
+import JavaImportQueries from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/model_providers/ImportAndQueries.java';
+
+Weaviate Embeddings' models can be accessed directly from a Weaviate Cloud instance.
+
+[Configure a Weaviate vector index](#configure-the-vectorizer) to use a Weaviate Embeddings model, and Weaviate will generate embeddings for various operations using the specified model and your Weaviate API key. This feature is called the *vectorizer*.
+
+At [import time](#data-import), Weaviate generates text object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts text queries into embeddings.
+
+
+
+## Requirements
+
+To use Weaviate Embeddings, you need:
+
+- A Weaviate Cloud instance running at least Weaviate `1.27.6`
+- To have [enabled "Weaviate Embeddings"](/developers/wcs/embeddings/index.md#administration) in your Weaviate Cloud organisation
+- A Weaviate client library that supports Weaviate Embeddings:
+ - Python client version `4.9.5` or higher
+ - JavaScript/TypeScript client version `3.2.5` or higher
+ - Go/Java clients are not yet officially supported; you must pass the `X-Weaviate-Api-Key` and `X-Weaviate-Cluster-Url` headers manually upon instantiation as shown below.
+
+### Weaviate configuration
+
+The Weaviate Embeddings vectorizer is only available for use by Weaviate Cloud instances. At this time, Weaviate Embeddings is not available for self-hosted users.
+
+### API credentials
+
+Weaviate Embeddings is integrated with Weaviate Cloud. Your Weaviate Cloud credentials will be used to authorize your Weaviate Cloud instance's access for Weaviate Embeddings.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Configure the vectorizer
+
+[Configure a Weaviate index](../../manage-data/collections.mdx#specify-a-vectorizer) as follows to use a Weaviate Embeddings model:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Select a model
+
+You can specify one of the [available models](#available-models) for the vectorizer to use, as shown in the following configuration example.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified.
+
+import VectorizationBehavior from '/_includes/vectorization.behavior.mdx';
+
+
+ Vectorization behavior
+
+
+
+
+
+### Vectorizer parameters
+
+The following examples show how to configure Weaviate Embeddings-specific options.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+- `model`: The name of the model to use for embedding generation.
+- `dimensions`: The number of dimensions to use for the generated embeddings. Only available for models that support Matryoshka Representation Learning.
+- `base_url`: The base URL for the Weaviate Embeddings service. (Not required in most cases.)
+
+## Data import
+
+After configuring the vectorizer, [import data](../../manage-data/import.mdx) into Weaviate. Weaviate generates embeddings for text objects using the specified model.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+:::tip Re-use existing vectors
+If you already have a compatible model vector available, you can provide it directly to Weaviate. This can be useful if you have already generated embeddings using the same model and want to use them in Weaviate, such as when migrating data from another system.
+:::
+
+## Searches
+
+Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified WED model.
+
+
+
+### Vector (near text) search
+
+When you perform a [vector search](../../search/similarity.md#search-with-text), Weaviate converts the text query into an embedding using the specified model and returns the most similar objects from the database.
+
+The query below returns the `n` most similar objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+### Hybrid search
+
+:::info What is a hybrid search?
+A hybrid search performs a vector search and a keyword (BM25) search, before [combining the results](../../search/hybrid.md#change-the-ranking-method) to return the best matching objects from the database.
+:::
+
+When you perform a [hybrid search](../../search/hybrid.md), Weaviate converts the text query into an embedding using the specified model and returns the best scoring objects from the database.
+
+The query below returns the `n` best scoring objects from the database, set by `limit`.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## References
+
+### Available models
+
+- `arctic-embed-m-v1.5` (default)
+ - A 109M parameter, 768-dimensional model for enterprise retrieval tasks in English.
+ - Trained with Matryoshka Representation Learning to allow vector truncation with minimal loss.
+ - Quantization-friendly: Using scalar quantization and 256 dimensions provides 99% of unquantized, full-precision performance.
+ - Read more at the [Snowflake blog](https://www.snowflake.com/engineering-blog/arctic-embed-m-v1-5-enterprise-retrieval/), and the Hugging Face [model card](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5)
+ - Allowable `dimensions`: 768 (default), 256
+
+Additional models will be added in the future.
+
+:::info Input truncation
+Currently, input exceeding the model's context windows is truncated from the right (i.e. the end of the input).
+:::
+
+## Further resources
+
+### Code examples
+
+Once the integrations are configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples:
+
+- The [how-to: manage data](../../manage-data/index.md) guides show how to perform data operations (i.e. create, update, delete).
+- The [how-to: search](../../search/index.md) guides show how to perform search operations (i.e. vector, keyword, hybrid) as well as retrieval augmented generation.
+
+### Further resources
+
+- Weaviate Embeddings [Documentation](/developers/wcs/embeddings)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/model-providers/weaviate/index.md b/developers/weaviate/model-providers/weaviate/index.md
new file mode 100644
index 0000000000..5fd705d078
--- /dev/null
+++ b/developers/weaviate/model-providers/weaviate/index.md
@@ -0,0 +1,50 @@
+---
+title: Weaviate Embeddings
+sidebar_position: 10
+image: og/docs/integrations/provider_integrations_wes.jpg
+# tags: ['model providers', 'weaviate', 'weaviate embeddings']
+---
+
+:::info Access to Weaviate Embeddings during technical preview
+Weaviate Embeddings is currently in free technical preview for Weaviate Cloud Serverless users.
+
+To try out Weaviate Embeddings at this time, please [contact us](https://events.weaviate.io/embeddings-preview) to request access.
+:::
+
+
+
+[Weaviate Embeddings](/developers/wcs/embeddings) offers vectorizer models to Weaviate Cloud users, allowing users to leverage Weaviate Embeddings' models directly from a Weaviate Cloud database instance.
+
+These integrations empower developers to build sophisticated AI-driven applications with ease.
+
+## Integrations with Weaviate Embeddings
+
+### Embedding models for semantic search
+
+
+
+Weaviate Embeddings' embedding models transform text data into high-dimensional vector representations, capturing semantic meaning and context.
+
+[Weaviate Cloud integrates with Weaviate Embeddings' embedding models](./embeddings.md) to enable seamless vectorization of data. This integration allows users to perform semantic and hybrid search operations without the need for additional preprocessing or data transformation steps.
+
+[Weaviate Embeddings integration page](./embeddings.md)
+
+## Summary
+
+These integrations enable developers to leverage Weaviate Embeddings' powerful models directly within Weaviate.
+
+In turn, they simplify the process of building AI-driven applications to speed up your development process, so that you can focus on creating innovative solutions.
+
+## Get started
+
+You must provide a valid Weaviate Cloud API key to Weaviate for these integrations. Go to [Weaviate Cloud](https://console.weaviate.cloud/) to sign up and obtain an API key.
+
+Then, go to the relevant integration page to learn how to configure Weaviate with the Weaviate Embeddings models and start using them in your applications.
+
+- [Text Embeddings](./embeddings.md)
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/modules/custom-modules.md b/developers/weaviate/modules/custom-modules.md
index 5507359307..cd878291b1 100644
--- a/developers/weaviate/modules/custom-modules.md
+++ b/developers/weaviate/modules/custom-modules.md
@@ -1,5 +1,6 @@
---
title: Custom modules
+description: Explore custom modules in Weaviate for personalized data management features.
sidebar_position: 90
image: og/docs/modules/custom-modules.jpg
# tags: ['modules', 'other modules', 'custom modules']
diff --git a/developers/weaviate/modules/img2vec-neural.md b/developers/weaviate/modules/img2vec-neural.md
index b7034517f1..82980f429e 100644
--- a/developers/weaviate/modules/img2vec-neural.md
+++ b/developers/weaviate/modules/img2vec-neural.md
@@ -9,9 +9,6 @@ image: og/docs/modules/img2vec-neural.jpg
For new projects, we recommend using the [Transformers multi-modal integration](../model-providers/transformers/embeddings-multimodal.md) module instead of `img2vec-neural`. This uses CLIP models, which uses a more modern model architecture than `resnet` models used in `img2vec-neural`. CLIP models are also multi-modal, meaning they can handle both images and text and therefore applicable to a wider range of use cases.
:::
-
-## Overview
-
The `img2vec-neural` module enables Weaviate to obtain vectors locally images using a [`resnet50`](https://arxiv.org/abs/1512.03385) model.
`img2vec-neural` encapsulates the model in a Docker container, which allows independent scaling on GPU-enabled hardware while keeping Weaviate on CPU-only hardware, as Weaviate is CPU-optimized.
@@ -68,7 +65,6 @@ services:
PERSISTENCE_DATA_PATH: "./data"
# highlight-start
ENABLE_MODULES: 'img2vec-neural'
- DEFAULT_VECTORIZER_MODULE: 'img2vec-neural'
IMAGE_INFERENCE_API: "http://i2v-neural:8080"
# highlight-end
CLUSTER_HOSTNAME: 'node1'
diff --git a/developers/weaviate/modules/ner-transformers.md b/developers/weaviate/modules/ner-transformers.md
index 38140f15b0..a215172ce7 100644
--- a/developers/weaviate/modules/ner-transformers.md
+++ b/developers/weaviate/modules/ner-transformers.md
@@ -1,5 +1,6 @@
---
title: Named Entity Recognition
+description: Integrate NER Transformers in Weaviate to identify and categorize entities in text.
sidebar_position: 60
image: og/docs/modules/ner-transformers.jpg
# tags: ['ner-transformers', 'transformers', 'token classification']
@@ -47,7 +48,6 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'text2vec-contextionary'
ENABLE_MODULES: 'text2vec-contextionary,ner-transformers'
CLUSTER_HOSTNAME: 'node1'
contextionary:
diff --git a/developers/weaviate/modules/qna-openai.md b/developers/weaviate/modules/qna-openai.md
index 900999b03b..8a2890accf 100644
--- a/developers/weaviate/modules/qna-openai.md
+++ b/developers/weaviate/modules/qna-openai.md
@@ -103,7 +103,6 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'text2vec-openai'
ENABLE_MODULES: 'text2vec-openai,qna-openai'
OPENAI_APIKEY: sk-foobar # For use with OpenAI. Setting this parameter is optional; you can also provide the key at runtime.
OPENAI_ORGANIZATION: your-orgname # For use with OpenAI. Setting this parameter is optional; you can also provide the key at runtime.
diff --git a/developers/weaviate/modules/qna-transformers.md b/developers/weaviate/modules/qna-transformers.md
index 4647015f9d..ece4101ce1 100644
--- a/developers/weaviate/modules/qna-transformers.md
+++ b/developers/weaviate/modules/qna-transformers.md
@@ -1,5 +1,6 @@
---
title: Question Answering - transformers
+description: Add QnA Transformers to Weaviate for accurate question answering and insights.
sidebar_position: 40
image: og/docs/modules/qna-transformers.jpg
# tags: ['qna', 'qna-transformers', 'transformers']
@@ -50,7 +51,6 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'text2vec-transformers'
ENABLE_MODULES: 'text2vec-transformers,qna-transformers'
CLUSTER_HOSTNAME: 'node1'
t2v-transformers:
diff --git a/developers/weaviate/modules/ref2vec-centroid.md b/developers/weaviate/modules/ref2vec-centroid.md
index 995ba810a0..1e0252ead2 100644
--- a/developers/weaviate/modules/ref2vec-centroid.md
+++ b/developers/weaviate/modules/ref2vec-centroid.md
@@ -1,5 +1,6 @@
---
title: Ref2Vec Centroid Vectorizer
+description: Enhance vector search with Ref2Vec Centroid for stronger data representations.
sidebar_position: 25
image: og/docs/modules/ref2vec-centroid.jpg
# tags: ['ref2vec', 'ref2vec-centroid', 'centroid']
diff --git a/developers/weaviate/modules/spellcheck.md b/developers/weaviate/modules/spellcheck.md
index a8ed46931c..3d6420925f 100644
--- a/developers/weaviate/modules/spellcheck.md
+++ b/developers/weaviate/modules/spellcheck.md
@@ -1,5 +1,6 @@
---
title: Spell Check
+description: Integrate spellcheck in Weaviate to improve text data quality and search accuracy.
sidebar_position: 70
image: og/docs/modules/text-spellcheck.jpg
# tags: ['modules', 'other modules', 'spellcheck']
@@ -45,7 +46,6 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'text2vec-contextionary'
ENABLE_MODULES: 'text2vec-contextionary,text-spellcheck'
CLUSTER_HOSTNAME: 'node1'
contextionary:
diff --git a/developers/weaviate/modules/sum-transformers.md b/developers/weaviate/modules/sum-transformers.md
index 67d3a35b47..62bdbf407a 100644
--- a/developers/weaviate/modules/sum-transformers.md
+++ b/developers/weaviate/modules/sum-transformers.md
@@ -1,5 +1,6 @@
---
title: Summarization
+description: Summarize data efficiently using the SUM Transformers module in Weaviate.
sidebar_position: 80
image: og/docs/modules/sum-transformers.jpg
# tags: ['transformers']
@@ -66,7 +67,6 @@ services:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
- DEFAULT_VECTORIZER_MODULE: 'text2vec-contextionary'
ENABLE_MODULES: 'text2vec-contextionary,sum-transformers'
CLUSTER_HOSTNAME: 'node1'
contextionary:
diff --git a/developers/weaviate/modules/text2vec-contextionary.md b/developers/weaviate/modules/text2vec-contextionary.md
index 98668f80b0..d767de79eb 100644
--- a/developers/weaviate/modules/text2vec-contextionary.md
+++ b/developers/weaviate/modules/text2vec-contextionary.md
@@ -1,14 +1,11 @@
---
title: "Contextionary Vectorizer"
+description: Use Text2Vec Contextionary in Weaviate for improved context-based text vectorization.
sidebar_position: 10
image: og/docs/modules/text2vec-contextionary.jpg
# tags: ['text2vec', 'text2vec-contextionary', 'contextionary']
---
-
-
-## Overview
-
The `text2vec-contextionary` module enables Weaviate to obtain vectors locally using a lightweight model.
Key notes:
@@ -79,7 +76,6 @@ services:
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
# highlight-start
ENABLE_MODULES: 'text2vec-contextionary'
- DEFAULT_VECTORIZER_MODULE: 'text2vec-contextionary'
# highlight-end
CLUSTER_HOSTNAME: 'node1'
# highlight-start
diff --git a/developers/weaviate/more-resources/faq.md b/developers/weaviate/more-resources/faq.md
index 75074259aa..b2b6185de8 100644
--- a/developers/weaviate/more-resources/faq.md
+++ b/developers/weaviate/more-resources/faq.md
@@ -526,7 +526,6 @@ Read more on SIGQUIT [here](https://en.wikipedia.org/wiki/Signal_(IPC)#SIGQUIT)
> AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
> PERSISTENCE_DATA_PATH: './data'
> ENABLE_MODULES: 'text2vec-contextionary'
-> DEFAULT_VECTORIZER_MODULE: 'text2vec-contextionary'
> AUTOSCHEMA_ENABLED: 'false'
> contextionary:
> environment:
diff --git a/developers/weaviate/more-resources/migration/archive.md b/developers/weaviate/more-resources/migration/archive.md
new file mode 100644
index 0000000000..89152b24e0
--- /dev/null
+++ b/developers/weaviate/more-resources/migration/archive.md
@@ -0,0 +1,906 @@
+---
+title: Archive
+sidebar_position: 90
+image: og/docs/more-resources.jpg
+# tags: ['migration']
+---
+
+# Migration Guide: Archive
+
+This page contains the migration guides for older versions of Weaviate. For the most recent migration guides, please refer to the parent [migration guide](./index.md) page.
+
+## Migration for version 1.19.0
+
+This version introduces `indexFilterable` and `indexSearchable` variables for the new text indexes, whose values will be set based on the value of `indexInverted`.
+
+Since filterable & searchable are separate indexes, filterable does not exist in Weaviate instances upgraded from pre-`v1.19` to `v1.19`. The missing `filterable` index can be created though on startup for all `text/text[]` properties if env variable `INDEX_MISSING_TEXT_FILTERABLE_AT_STARTUP` is set.
+
+## Changelog for version v1.9.0
+
+* no breaking changes
+
+* *New Features*
+ * ### First Multi-modal module: CLIP Module (#1756, #1766)
+ This release [introduces the `multi2vec-clip` integration](/developers/weaviate/model-providers/transformers/embeddings-multimodal.md), a module that allows for multi-modal vectorization within a single vector space. A class can have `image` or `text` fields or both. Similarly, the module provides both a `nearText` and a `nearImage` search and allows for various search combinations, such as text-search on image-only content and various other combinations.
+
+ #### How to use
+
+ The following is a valid payload for a class that vectorizes both images and text fields:
+ ```json
+ {
+ "class": "ClipExample",
+ "moduleConfig": {
+ "multi2vec-clip": {
+ "imageFields": [
+ "image"
+ ],
+ "textFields": [
+ "name"
+ ],
+ "weights": {
+ "textFields": [0.7],
+ "imageFields": [0.3]
+ }
+ }
+ },
+ "vectorIndexType": "hnsw",
+ "vectorizer": "multi2vec-clip",
+ "properties": [
+ {
+ "dataType": [
+ "text"
+ ],
+ "name": "name"
+ },
+ {
+ "dataType": [
+ "blob"
+ ],
+ "name": "image"
+ }
+ ]
+ }
+ ```
+
+ Note that:
+ - `imageFields` and `textFields` in `moduleConfig.multi2vec-clip` do not both need to be set. However at least one of both must be set.
+ - `weights` in `moduleConfig.multi2vec-clip` is optional. If only a single property the property takes all the weight. If multiple properties exist and no weights are specified, the properties are equal-weighted.
+
+ You can then import data objects for the class as usual. Fill the `text` or `string` fields with text and/or fill the `blob` fields with a base64-encoded image.
+
+ #### Limitations
+ * As of `v1.9.0` the module requires explicit creation of a class. If you rely on auto-schema to create the class for you, it will be missing the required configuration about which fields should be vectorized. This will be addressed in a future release.
+
+* *Fixes*
+ * fix an error where deleting a class with `geoCoordinates` could lead to a panic due to missing cleanup (#1730)
+ * fix an issue where an error in a module would not be forwarded to the user (#1754)
+ * fix an issue where a class could not be deleted on some file system (e.g. AWS EFS) (#1757)
+
+
+## Migration to version v1.8.0
+
+### Migration Notice
+
+Version `v1.8.0` introduces multi-shard indexes and horizontal scaling. As a
+result the dataset needs to be migrated. This migration is performed automatically -
+without user interaction - when first starting up with Weaviate version
+`v1.8.0`. However, it cannot be reversed. We, therefore, recommend carefully
+reading the following migration notes and making a case-by-case decision about the
+best upgrade path for your needs.
+
+#### Why is a data migration necessary?
+
+Prior to `v1.8.0` Weaviate did not support multi-shard indexes. The feature was
+already planned, therefore data was already contained in a single shard with a
+fixed name. A migration is necessary to move the data from a single fixed shard
+into a multi-shard setup. The amount of shards is not changed. When you run
+`v1.8.0` on a dataset the following steps happen automatically:
+
+* Weaviate discovers the missing sharding configuration for your classes and fills it with the default values
+* When shards start-up and they do not exist on disk, but a shard with a fixed
+ name from `v1.7.x` exists, Weaviate automatically recognizes that a migration
+ is necessary and moves the data on disk
+* When Weaviate is up and running the data has been migrated.
+
+**Important Notice:** As part of the migration Weaviate will assign the shard
+to the (only) node available in the cluster. You need to make sure that this
+node has a stable hostname. If you run on Kubernetes, hostnames are stable
+(e.g. `weaviate-0` for the first node). However with `Docker Compose` hostnames
+default to the id of the container. If you remove your containers (e.g.
+`docker compose down`) and start them up again, the hostname will have changed.
+This will lead to errors where Weaviate mentions that it cannot find the node
+that the shard belongs to. The node sending the error message is the node that
+owns the shard itself, but it cannot recognize it, since its own name has
+changed.
+
+To remedy this, you can set a stable hostname **before starting up with
+v1.8.0** by setting the env var `CLUSTER_HOSTNAME=node1`. The actual name does
+not matter, as long as it's stable.
+
+If you forgot to set a stable hostname and are now running into the error
+mentioned above, you can still explicitly set the hostname that was used before
+which you can derive from the error message.
+
+Example:
+
+If you see the error message `"shard Knuw6a360eCY: resolve node name
+\"5b6030dbf9ea\" to host"`, you can make Weaviate usable again, by setting
+`5b6030dbf9ea` as the host name: `CLUSTER_HOSTNAME=5b6030dbf9ea`.
+
+#### Should you upgrade or reimport?
+
+In addition to new features, `v1.8.0` also contains a large collection of
+bugfixes. Some of the bugs affect how Weaviate writes the HNSW index to disk.
+A pre-1.18.0 index on disk may not be as good a freshly built `v1.8.0` index.
+If you can import using a script, we generally recommend starting with a fresh
+`v1.8.0` instance and reimporting instead of migrating.
+
+#### Is downgrading possible after upgrading?
+
+Note that the data migration which happens at the first startup of v1.8.0 is
+not automatically reversible. If you plan on downgrading to `v1.7.x` again
+after upgrading, you must explicitly create a backup of the state prior to
+upgrading.
+
+### Changelog
+
+
+## Changelog for version v1.7.2
+* No breaking changes
+* New features
+ * ### Array Datatypes (#1691)
+ Added `boolean[]` and `date[]`.
+ * ### Make property names less strict (#1562)
+ Property names in a data schema allows: `/[_A-Za-z][_0-9A-Za-z]*/`. i.e. it will allow for using underscores, will allow numbers and will fix the issue about trailing upper-case characters. But it won't allow for many other special characters, such as dash (-) or language-specific characters like Umlauts, etc, due to GraphQL restrictions.
+* Bug fixes
+ * ### Aggregation on array data type (#1686)
+
+
+## Changelog for version v1.7.0
+* No breaking changes
+* New features
+ * ### Array Datatypes (#1611)
+ Starting with this release, primitive object properties are no longer limited to individual properties, but can also include lists of primitives. Array types can be stored, filtered and aggregated in the same way as other primitives.
+
+ Auto-schema will automatically recognize lists of `string`/`text` and `number`/`int`. You can also explicitly specify lists in the schema by using the following data types `string[]`, `text[]`, `int[]`, `number[]`. A type that is assigned to be an array, must always stay an array, even if it only contains a single element.
+
+ * ### New Module: `text-spellcheck` - Check and autocorrect misspelled search terms (#1606)
+ Use the new spellchecker module to verify user-provided search queries (in existing `nearText` or `ask` functions) are spelled correctly and even suggest alternative, correct spellings. Spell-checking happens at query time.
+
+ There are two ways to use this module:
+ 1. It provides a new additional property which can be used to check (but not alter) the provided queries:
+ The following query:
+ ```graphql
+ {
+ Get {
+ Post(nearText: {
+ concepts: "missspelled text"
+ }) {
+ content
+ _additional {
+ spellCheck {
+ changes {
+ corrected
+ original
+ }
+ didYouMean
+ location
+ originalText
+ }
+ }
+ }
+ }
+ }
+ ```
+
+ will produce results similar to the following:
+
+ ```
+ "_additional": {
+ "spellCheck": [
+ {
+ "changes": [
+ {
+ "corrected": "misspelled",
+ "original": "missspelled"
+ }
+ ],
+ "didYouMean": "misspelled text",
+ "location": "nearText.concepts[0]",
+ "originalText": "missspelled text"
+ }
+ ]
+ },
+ "content": "..."
+ },
+ ```
+ 2. It extends existing `text2vec-*` modules with an `autoCorrect` flag, which can be used to automatically correct the query if misspelled.
+
+ * ### New Module `ner-transformers` - Extract entities from Weaviate using transformers (#1632)
+ Use transformer-based models to extract entities from your existing Weaviate objects on the fly. Entity extraction happens at query time. Note that for maximum performance, transformer-based models should run with GPUs. CPUs can be used, but the throughput will be lower.
+
+ To make use of the module's capabilities, simply extend your query with the following new `_additional` property:
+
+ ```graphql
+ {
+ Get {
+ Post {
+ content
+ _additional {
+ tokens(
+ properties: ["content"], # is required
+ limit: 10, # optional, int
+ certainty: 0.8 # optional, float
+ ) {
+ certainty
+ endPosition
+ entity
+ property
+ startPosition
+ word
+ }
+ }
+ }
+ }
+ }
+
+ ```
+ It will return results similar to the following:
+
+ ```
+ "_additional": {
+ "tokens": [
+ {
+ "property": "content",
+ "entity": "PER",
+ "certainty": 0.9894614815711975,
+ "word": "Sarah",
+ "startPosition": 11,
+ "endPosition": 16
+ },
+ {
+ "property": "content",
+ "entity": "LOC",
+ "certainty": 0.7529033422470093,
+ "word": "London",
+ "startPosition": 31,
+ "endPosition": 37
+ }
+ ]
+ }
+ ```
+* Bug fixes
+ * Aggregation can get stuck when aggregating `number` datatypes (#1660)
+
+## Changelog for version 1.6.0
+* No breaking changes
+* No new features
+ * **Zero Shot Classification (#1603)** This release adds a new classification type `zeroshot` that works with any `vectorizer` or custom vectors. It picks the label objects that have the lowest distance to the source objects. The link is made using cross-references, similar to existing classifications in Weaviate. To start a `zeroshot` classification use `"type": "zeroshot"` in your `POST /v1/classficiations` request and specify the properties you want classified normally using `"classifyProperties": [...]`. As zero shot involves no training data, you cannot set `trainingSetWhere` filters, but can filter both source (`"sourceWhere"`) and label objects (`"targetWhere"`) directly.
+* Bug fixes
+
+
+## Changelog for version 1.5.2
+
+* No breaking changes
+* No new features
+* Bug fixes:
+* ### Fix possible data races (`short write`) (#1643)
+ This release fixes various possible data races that could in the worst case lead to an unrecoverable error `"short write"`. The possibility for those races was introduced in `v.1.5.0` and we highly recommend anyone running on the `v1.5.x` timeline to upgrade to `v1.5.2` immediately.
+
+## Changelog for version 1.5.1
+
+* No breaking changes
+* No new features
+* Bug fixes:
+* ### Crashloop after unexpected crash in HNSW commit log (#1635)
+ If Weaviate was killed (e.g. OOMKill) while writing the commit log, it could not be parsed after the next restart anymore, thus ending up in a crashloop. This fix removes this. Note that no data will be lost on such a crash: The partially written commit log has not yet been acknowledged to the user, so no write guarantees have been given yet. It is therefore safe to discard.
+
+* ### Chained Like operator not working (#1638)
+ Prior to this fix, when chaining `Like` operators in `where` filters where each `valueString` or `valueText` contained a wildcard (`*`), typically only the first operator's results where reflected. This fix makes sure that the chaining (`And` or `Or`) is reflected correctly. This bug did not affect other operators (e.g. `Equal`, `GreaterThan`, etc) and only affected those `Like` queries where a wildcard was used.
+
+* ### Fix potential data race in Auto Schema features (#1636)
+ This fix improves incorrect synchronization on the auto schema feature which in extreme cases could lead to a data race.
+
+## Migration to version 1.5.0
+
+### Migration Notice
+*This release does not contain any API-level breaking changes, however, it changes the entire storage mechanism inside Weaviate. As a result, an in-place update is not possible. When upgrading from previous versions, a new setup needs to be created and all data reimported. Prior backups are not compatible with this version.*
+
+### Changelog
+* No breaking changes
+* New Features:
+ * *LSM-Tree based Storage*. Previous releases of Weaviate used a B+Tree based storage mechanism. This was not fast enough to keep up with the high write speed requirements of a large-scale import. This release completely rewrites the storage layer of Weaviate to use a custom LSM-tree approach. This leads to considerably faster import times, often more than 100% faster than the previous version.
+ * *Auto-Schema Feature*. Import data objects without creating a schema prior to import. The classes will be created automatically, they can still be adjusted manually. Weaviate will guess the property type based on the first time it sees a property. The defaults can be configured using the environment variables outlined in #1539. The feature is on by default, but entirely non-breaking. You can still create an explicit schema at will.
+* Fixes:
+ * *Improve Aggregation Queries*. Reduces the amount of allocations required for some aggregation queries, speeding them up and reduces the amount of timeouts encountered during aggregations.
+
+
+Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.5.0) for all the changes.
+
+
+## Changelog for version 1.4.0
+
+* No breaking changes
+* New Features:
+ * Image Module [`img2vec-neural`](/developers/weaviate/modules/img2vec-neural.md)
+ * Add Hardware acceleration for `amd64` CPUs (Intel, AMD)
+ * Support `arm64` technology for entire Weaviate stack
+ * Set `ef` at search time
+ * Introduce new dataType `blob`
+ * Skip vector-indexing a class
+* Fixes:
+ * Various Performance Fixes around the HNSW Vector Index
+ * Make property order consistent when vectorizing
+ * Fix issues around `PATCH` API when using custom vectors
+ * Detect schema settings that will most likely lead to duplicate vectors and print warning
+ * Fix missing schema validation on transformers module
+
+Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.4.0) for all the changes.
+
+
+## Changelog for version 1.3.0
+
+* No breaking changes
+* New feature: [Question Answering (Q&A) Module](/developers/weaviate/modules/qna-transformers.md)
+* New feature: New Meta Information for all transformer-based modules
+
+Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.3.0) for all the changes.
+
+## Changelog for version 1.2.0
+
+* No breaking changes
+* New feature: Introduction of the [Transformer Module](/developers/weaviate/modules/qna-transformers.md)
+
+Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.2.0) for all the changes.
+
+## Changelog for version 1.1.0
+
+* No breaking changes
+* New feature: GraphQL `nearObject` search to get most similar object.
+* Architectural update: Cross-reference batch import speed improvements.
+
+Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.1.0) for all the changes.
+
+## Migration to version 1.0.0
+
+Weaviate version 1.0.0 was released on 12 January 2021, and consists of the major update of modularization. From version 1.0.0, Weaviate is modular, meaning that the underlying structure relies on a *pluggable* vector index, *pluggable* vectorization modules with possibility to extend with *custom* modules.
+
+Weaviate release 1.0.0 from 0.23.2 comes with a significant amount of breaking changes in the data schema, API and clients. Here is an overview of all (breaking) changes.
+
+For client library specific changes, take a look at the change logs of the specific client ([Go](/developers/weaviate/client-libraries/go.md#change-logs), [Python](/developers/weaviate/client-libraries/python/index.md#change-logs) and [TypeScript/JavaScript](/developers/weaviate/client-libraries/typescript/index.mdx#changelog).
+
+Moreover, a new version of the Console is released. Visit the Console documentation for more information.
+
+### Summary
+This contains most overall changes, but not all details. Those are documented in ["Changes"](#changes).
+
+#### All RESTful API changes
+* from `/v1/schema/things/{ClassName}` to `/v1/schema/{ClassName}`
+* from `/v1/schema/actions/{ClassName} `to `/v1/schema/{ClassName}`from `/v1/schema/actions/{ClassName} `to `/v1/schema/{ClassName}`
+* from `/v1/things` to `/v1/objects`
+* from `/v1/actions` to `/v1/objects`
+* from `/v1/batching/things` to `/v1/batch/objects`
+* from `/v1/batching/actions` to `/v1/batch/objects`
+* from `/v1/batching/references` to `/v1/batch/references`
+* Additional data object properties are grouped in `?include=...` and the leading underscore of these properties is removed
+* The `/v1/modules/` endpoint is introduced.
+* the `/v1/meta/` endpoint now contains module specific information in `"modules"`
+
+#### All GraphQL API changes
+* Removal of Things and Actions layer in query hierarchy
+* Reference properties of data objects are lowercase (previously uppercased)
+* Underscore properties, uuid and certainty are now grouped in the object `_additional`
+* `explore()` filter is renamed to `near` filter
+* `nearVector(vector:[])` filter is introduced in `Get{}` query
+* `Explore (concepts: ["foo"]){}` query is changed to `Explore (near: ... ) {}`.
+
+#### All data schema changes
+* Removal of Things and Actions
+* Per class and per property configuration is changed to support modules and vector index type settings.
+
+#### All data object changes
+* From `schema` to `properties` in the data object.
+
+#### Contextionary
+* Contextionary is renamed to the module `text2vec-contextionary`
+* `/v1/c11y/concepts` to `/v1/modules/text2vec-contextionary/concepts`
+* `/v1/c11y/extensions` to `/v1/modules/text2vec-contextionary/extensions`
+* `/v1/c11y/corpus` is removed
+
+#### Other
+* Removal of `/things` and `/actions` in short and long beacons
+* Classification body is changed to support modularization
+* `DEFAULT_VECTORIZER_MODULE` is a new environment variable
+
+### Changes
+
+#### Removal of Things and Actions
+`Things` and `Actions` are removed from the Data Schema. This comes with the following changes in the schema definition and API endpoints:
+1. **Data schema:** The `semantic kind` (`Things` and `Actions`) is removed from the Schema Endpoint. This means the URLs will change:
+ * from `/v1/schema/things/{ClassName}` to `/v1/schema/{ClassName}`
+ * from `/v1/schema/actions/{ClassName} `to `/v1/schema/{ClassName}`
+1. **Data RESTful API endpoint:** The `semantic kind` (`Things` and `Actions`) is removed from the data Endpoint. Instead it will be namespaced as `/objects`. This means the URLs will change:
+ * from `/v1/things` to `/v1/objects`
+ * from `/v1/actions` to `/v1/objects`
+ * from `/v1/batching/things`to `/v1/batch/objects` (see also the [change in batching](#renaming-batching-to-batch))
+ * from `/v1/batching/actions`to `/v1/batch/objects` (see also the [change in batching](#renaming-batching-to-batch))
+1. **GraphQL:** The `Semantic Kind` "level" in the query hierarchy will be removed without replacement (In `Get` and `Aggregate` queries), i.e.
+ ```graphql
+ {
+ Get {
+ Things {
+ ClassName {
+ propName
+ }
+ }
+ }
+ }
+ ```
+
+ will become
+
+ ```graphql
+ {
+ Get {
+ ClassName {
+ propName
+ }
+ }
+ }
+ ```
+1. **Data Beacons:** The `Semantic Kind` will be removed from beacons:
+ * **Short-form Beacon:**
+
+ * `weaviate://localhost/things/4fbacd6e-1153-47b1-8cb5-f787a7f01718`
+
+ to
+
+ * `weaviate://localhost/4fbacd6e-1153-47b1-8cb5-f787a7f01718`
+
+ * **Long-form Beacon:**
+
+ * `weaviate://localhost/things/ClassName/4fbacd6e-1153-47b1-8cb5-f787a7f01718/propName`
+
+ to
+
+ * `weaviate://localhost/ClassName/4fbacd6e-1153-47b1-8cb5-f787a7f01718/propName`
+
+#### Renaming /batching/ to /batch/
+
+* `/v1/batching/things` to `/v1/batch/objects`
+* `/v1/batching/actions` to `/v1/batch/objects`
+* `/v1/batching/references` to `/v1/batch/references`
+
+
+#### From "schema" to "properties" in data object
+
+The name "schema" on the data object is not intuitive and is replaced by "properties". The change looks like:
+
+```json
+{
+ "class": "Article",
+ "schema": {
+ "author": "Jane Doe"
+ }
+}
+```
+
+to
+
+```json
+{
+ "class": "Article",
+ "properties": {
+ "author": "Jane Doe"
+ }
+}
+```
+
+#### Consistent casing in GraphQL properties
+
+Previously, reference properties in the schema definitions are always lowercase, yet in graphQL they needed to be uppercased. E.g.: `Article { OfAuthor { … on Author { name } } } }`, even though the property is defined as ofAuthor. New is that the casing in GraphQL reflects exactly the casing in the schema definition, thus the above example would become: `Article { ofAuthor { … on Author { name } } } }`
+
+#### Additional data properties in GraphQL and RESTful API
+Since modularization, a module can contribute to the additional properties of a data object (thus are not fixed), which should be retrievable by the GraphQL and/or RESTful API.
+1. **REST**: `additional` properties (formerly named `"underscore"` properties) can be included in RESTful query calls like `?include=...`, e.g. `?include=classification`. The underscores will thus be removed from the names (e.g. `?include=_classification` is deprecated). In the Open API specifications, all additional properties will be grouped in the object `additional`. For example:
+ ```json
+ {
+ "class": "Article",
+ "schema": { ... },
+ "_classification": { … }
+ }
+ ```
+
+ to
+
+ ```json
+ {
+ "class": "Article",
+ "properties": { ... },
+ "additional": {
+ "classification": { ... }
+ }
+ }
+ ```
+2. **GraphQL**: `"underscore"` properties are renamed to `additional` properties in GraphQL queries.
+ 1. All former `"underscore"` properties of a data object (e.g. `_certainty`) are now grouped in the `_additional {}` object (e.g. `_additional { certainty } `).
+ 2. The `uuid` property is now also placed in the `_additional {}` object and renamed to `id` (e.g. `_additional { id } `).
+ This example covers both changes:
+
+ From
+
+ ```graphql
+ {
+ Get {
+ Things {
+ Article {
+ title
+ uuid
+ certainty
+ _classification
+ }
+ }
+ }
+ }
+ ```
+
+ to
+
+ ```graphql
+ {
+ Get {
+ Article {
+ title
+ _additional { # leading _ prevents clashes
+ certainty
+ id # replaces uuid
+ classification
+ }
+ }
+ }
+ }
+ ```
+
+#### Modules RESTful endpoint
+With the modularization of Weaviate, the `v1/modules/` endpoint is introduced.
+
+#### GraphQL semantic search
+
+With the modularization, it becomes possible to vectorize non-text objects. Search is no longer restricted to use the Contextionary's vectorization of text and data objects, but could also be applied to non-text objects or raw vectors. The formerly 'explore' filter in Get queries and 'Explore' queries in GraphQL were tied to text, but the following changes are made to this filter with the new version of Weaviate:
+
+1. The filter `Get ( explore: {} ) {}` is renamed to `Get ( near: {} ) {}`.
+ 1. New: `Get ( nearVector: { vector: [.., .., ..] } ) {}` is module independent and will thus always be available.
+ 2. `Get ( explore { concepts: ["foo"] } ) {}` will become `Get ( nearText: { concepts: ["foo"] } ) {}` and is only available if the `text2vec-contextionary` module is attached.
+
+ From
+
+ ```graphql
+ {
+ Get {
+ Things {
+ Article (explore: { concepts: ["foo"] } ) {
+ title
+ }
+ }
+ }
+ }
+ ```
+
+ to
+
+ ```graphql
+ {
+ Get {
+ Article (near: ... ) {
+ title
+ }
+ }
+ }
+ ```
+
+2. Similarly to the explore sorter that is used in the `Get {}` API, the `Explore {}` API also assumes text. The following change is applied:
+
+ From
+
+ ```graphql
+ {
+ Explore (concepts: ["foo"]) {
+ beacon
+ }
+ }
+ ```
+
+ to
+
+ ```graphql
+ {
+ Explore (near: ... ) {
+ beacon
+ }
+ }
+ ```
+
+#### Data schema configuration
+1. **Per-class configuration**
+
+ With modularization, it is possible to configure per class the vectorizer module, module-specific configuration for the overall class, vector index type, and vector index type specific configuration:
+ * The `vectorizer` indicates which module (if any) are responsible for vectorization.
+ * The `moduleConfig` allows configuration per module (by name).
+ * See [here](#text2vec-contextionary) for Contextionary specific property configuration.
+ * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/vector-index.md#hnsw))
+ * The `vectorIndexConfig` is an arbitrary object passed to the index for config (defaults can be found [here](/developers/weaviate/config-refs/schema/vector-index.md#how-to-configure-hnsw) )
+
+ All changes are in this example:
+
+ ```json
+ {
+ "class": "Article",
+ "vectorizeClassName": true,
+ "description": "string",
+ "properties": [ … ]
+ }
+ ```
+
+ will become
+
+ ```json
+ {
+ "class": "Article",
+ "vectorIndexType": "hnsw", # defaults to hnsw
+ "vectorIndexConfig": {
+ "efConstruction": 100
+ },
+ "moduleConfig": {
+ "text2vec-contextionary": {
+ "vectorizeClassName": true
+ },
+ "encryptor5000000": { "enabled": true } # example
+ },
+ "description": "string",
+ "vectorizer": "text2vec-contextionary", # default is configurable
+ "properties": [ … ]
+ }
+ ```
+
+2. **Per-property configuration**
+
+ With modularization, it is possible to configure per property module-specific configuration per property if available and it can be specified if a property should be included in the inverted index.
+ * The `moduleConfig` allows configuration per module (by name).
+ * See [here](#text2vec-contextionary) for Contextionary specific property configuration.
+ * `index` will become `indexInverted`: a boolean that indicates whether a property should be indexed in the inverted index.
+
+ All changes are in this example:
+
+ ```json
+ {
+ "dataType": [ "text" ],
+ "description": "string",
+ "cardinality": "string",
+ "vectorizePropertyName": true,
+ "name": "string",
+ "keywords": [ … ],
+ "index": true
+ }
+ ```
+
+ will become
+
+ ```json
+ {
+ "dataType": [ "text" ],
+ "description": "string",
+ "moduleConfig": {
+ "text2vec-contextionary": {
+ "skip": true,
+ "vectorizePropertyName": true,
+ }
+ },
+ "name": "string",
+ "indexInverted": true
+ }
+ ```
+
+#### RESTful /meta endpoint
+
+The `/v1/meta` object now contains module specific information at in the newly introduced namespaced `modules.` property:
+
+From
+
+```json
+{
+ "hostname": "string",
+ "version": "string",
+ "contextionaryWordCount": 0,
+ "contextionaryVersion": "string"
+}
+```
+
+to
+
+```json
+{
+ "hostname": "string",
+ "version": "string",
+ "modules": {
+ "text2vec-contextionary": {
+ "wordCount": 0,
+ "version": "string"
+ }
+ }
+}
+```
+
+#### Modular classification
+
+Some classification types are tied to modules (e.g. the former "contextual" classification is tied to the `text2vec-contextionary` module. We make a distinction between fields which are always present and those which are type dependent. Additionally the API is improved by grouping `settings` and `filters` in separate properties. kNN classification is the only type of classification that is present with Weaviate Core without dependency on modules. The former "contextual" classification is tied to the `text2vec-contextionary` module, see [here](#text2vec-contextionary). An example of how the change looks like in the classification API POST body:
+
+From
+
+```json
+{
+ "class": "City",
+ "classifyProperties": ["inCountry"],
+ "basedOnProperties": ["description"],
+ "type": "knn",
+ "k": 3,
+ "sourceWhere": { … },
+ "trainingSetWhere": { … },
+ "targetWhere": { … },
+}
+
+```
+
+To
+
+```json
+{
+ "class": "City",
+ "classifyProperties": ["inCountry"],
+ "basedOnProperties": ["description"],
+ "type": "knn",
+ "settings": {
+ "k": 3
+ },
+ "filters": {
+ "sourceWhere": { … },
+ "trainingSetWhere": { … },
+ "targetWhere": { … },
+ }
+}
+```
+
+And the API GET body:
+
+From
+
+```json
+{
+ "id": "ee722219-b8ec-4db1-8f8d-5150bb1a9e0c",
+ "class": "City",
+ "classifyProperties": ["inCountry"],
+ "basedOnProperties": ["description"],
+ "status": "running",
+ "meta": { … },
+ "type": "knn",
+ "k": 3,
+ "sourceWhere": { … },
+ "trainingSetWhere": { … },
+ "targetWhere": { … },
+}
+```
+
+To
+
+```json
+{
+ "id": "ee722219-b8ec-4db1-8f8d-5150bb1a9e0c",
+ "class": "City",
+ "classifyProperties": ["inCountry"],
+ "basedOnProperties": ["description"],
+ "status": "running",
+ "meta": { … },
+ "type": "knn",
+ "settings": {
+ "k": 3
+ },
+ "filters": {
+ "sourceWhere": { … },
+ "trainingSetWhere": { … },
+ "targetWhere": { … },
+ }
+}
+```
+
+#### text2vec-contextionary
+The Contextionary becomes the first vectorization module of Weaviate, renamed to `text2vec-contextionary` in formal use. This brings the following changes:
+1. **RESTful** endpoint `/v1/c11y` changes to `v1/modules/text2vec-contextionary`:
+ * `/v1/c11y/concepts` to `/v1/modules/text2vec-contextionary/concepts`
+ * `/v1/c11y/extensions` to `/v1/modules/text2vec-contextionary/extensions`
+ * `/v1/c11y/corpus` is removed
+2. **Data schema:** `text2vec-contextionary`-specific module configuration options in the schema definition
+ 1. **Per-class**. `"vectorizeClassName"` indicates whether the class name should be taken into the vector calculation of data objects.
+
+ ```json
+ {
+ "class": "Article",
+ "moduleConfig": {
+ "text2vec-contextionary": {
+ "vectorizeClassName": true
+ }
+ },
+ "description": "string",
+ "vectorizer": "text2vec-contextionary",
+ "properties": [ … ]
+ }
+ ```
+
+ 2. **Per-property.** `skip` tells whether to skip the entire property (including value) from the vector position of the data object. `vectorizePropertyName` indicates whether the property name should be taken into the vector calculation of data objects.
+
+ ```json
+ {
+ "dataType": [ "text" ],
+ "description": "string",
+ "moduleConfig": {
+ "text2vec-contextionary": {
+ "skip": true,
+ "vectorizePropertyName": true,
+ }
+ },
+ "name": "string",
+ "indexInverted": true
+ }
+ ```
+3. **Contextual classification**. Contextual classification is dependent on the module `text2vec-contextionary`. It can be activated in `/v1/classifications/` the following with the classification name `text2vec-contextionary-contextual`:
+
+From
+
+```json
+{
+ "class": "City",
+ "classifyProperties": ["inCountry"],
+ "basedOnProperties": ["description"],
+ "type": "contextual",
+ "informationGainCutoffPercentile": 30,
+ "informationGainMaximumBoost": 3,
+ "tfidfCutoffPercentile": 80,
+ "minimumUsableWords": 3,
+ "sourceWhere": { … },
+ "trainingSetWhere": { … },
+ "targetWhere": { … },
+}
+```
+
+To
+
+```json
+{
+ "class": "City",
+ "classifyProperties": ["inCountry"],
+ "basedOnProperties": ["description"],
+ "type": "text2vec-contextionary-contextual",
+ "settings": {
+ "informationGainCutoffPercentile": 30,
+ "informationGainMaximumBoost": 3,
+ "tfidfCutoffPercentile": 80,
+ "minimumUsableWords": 3
+ },
+ "filters": {
+ "sourceWhere": { … },
+ "trainingSetWhere": { … },
+ "targetWhere": { … }
+ }
+}
+```
+
+#### Default vectorizer module
+The default vectorizer module can be specified in a new environment variable so that this doesn't have to be specified on every data class in the schema. The environment variable is `DEFAULT_VECTORIZER_MODULE`, which can be set to for example `DEFAULT_VECTORIZER_MODULE="text2vec-contextionary"`.
+
+
+### Official release notes
+Official release notes can be found on [GitHub](https://github.com/weaviate/weaviate/releases/tag/0.23.0).
+
+## Questions and feedback
+
+import DocsFeedback from '/_includes/docs-feedback.mdx';
+
+
diff --git a/developers/weaviate/more-resources/migration/index.md b/developers/weaviate/more-resources/migration/index.md
index 6ad50decab..28ff4e1dc3 100644
--- a/developers/weaviate/more-resources/migration/index.md
+++ b/developers/weaviate/more-resources/migration/index.md
@@ -5,898 +5,59 @@ image: og/docs/more-resources.jpg
# tags: ['migration']
---
+## Upgrades
-## Migration for version 1.19.0
+Weaviate is under active development, with new features and improvements being added regularly, including bugfixes. To take advantage of these updates, we recommend upgrading your Weaviate instance regularly.
-This version introduces `indexFilterable` and `indexSearchable` variables for the new text indexes, whose values will be set based on the value of `indexInverted`.
+### General upgrade instructions
-Since filterable & searchable are separate indexes, filterable does not exist in Weaviate instances upgraded from pre-`v1.19` to `v1.19`. The missing `filterable` index can be created though on startup for all `text/text[]` properties if env variable `INDEX_MISSING_TEXT_FILTERABLE_AT_STARTUP` is set.
+When upgrading Weaviate, we recommend that you:
-## Changelog for version v1.9.0
+1. Create a complete [backup](../../configuration/backups.md) of your current Weaviate instance before beginning any upgrade process.
+1. Plan to upgrade one minor version at a time, always using the latest patch version of each minor release.
-* no breaking changes
+This approach of upgrading one minor version at a time helps to minimize the risk of issues during the upgrade process, by mirroring our testing and release process. Upgrading to the latest patch version of each minor release ensures that you have the latest bugfixes and improvements.
-* *New Features*
- * ### First Multi-modal module: CLIP Module (#1756, #1766)
- This release [introduces the `multi2vec-clip` integration](/developers/weaviate/model-providers/transformers/embeddings-multimodal.md), a module that allows for multi-modal vectorization within a single vector space. A class can have `image` or `text` fields or both. Similarly, the module provides both a `nearText` and a `nearImage` search and allows for various search combinations, such as text-search on image-only content and various other combinations.
+### Version-specific migration guides
- #### How to use
+- When upgrading to version `1.25.x` from `1.24.x` (or lower), you must perform a [Raft migration](#raft-migration-v1250).
+- When upgrading to version `1.26.x` or higher (from the preceding version), ensure that the cluster metadata is synchronized.
+ - To do so, poll the `/cluster/statistics` endpoint, and check that the correct number of nodes are reporting statistics, and the `synchronized` flag is showing `true`, before proceeding with the upgrade.
+ - For an example implementation, see the [`wait_for_raft_sync` function here](https://github.com/weaviate/weaviate-local-k8s/blob/main/utilities/helpers.sh).
- The following is a valid payload for a class that vectorizes both images and text fields:
- ```json
- {
- "class": "ClipExample",
- "moduleConfig": {
- "multi2vec-clip": {
- "imageFields": [
- "image"
- ],
- "textFields": [
- "name"
- ],
- "weights": {
- "textFields": [0.7],
- "imageFields": [0.3]
- }
- }
- },
- "vectorIndexType": "hnsw",
- "vectorizer": "multi2vec-clip",
- "properties": [
- {
- "dataType": [
- "text"
- ],
- "name": "name"
- },
- {
- "dataType": [
- "blob"
- ],
- "name": "image"
- }
- ]
- }
- ```
+:::tip Scenario: upgrading from `v1.25.10` to `v1.27`
- Note that:
- - `imageFields` and `textFields` in `moduleConfig.multi2vec-clip` do not both need to be set. However at least one of both must be set.
- - `weights` in `moduleConfig.multi2vec-clip` is optional. If only a single property the property takes all the weight. If multiple properties exist and no weights are specified, the properties are equal-weighted.
+Between `v1.25` and `v1.27`, there are two minor versions, `v1.26` and `v1.27`. So:
+
- You can then import data objects for the class as usual. Fill the `text` or `string` fields with text and/or fill the `blob` fields with a base64-encoded image.
+1. Create a backup of your current Weaviate instance.
+1. Go to the [Weaviate releases page](https://github.com/weaviate/weaviate/tags):
+ 1. Find the latest `v1.26` patch version (e.g.: `1.26.11`).
+ 1. Find the latest `v1.27` patch version (e.g.: `1.27.5`).
+1. Upgrade to the latest patch version of `v1.26`.
+1. Upgrade to the latest patch version of `v1.27`.
- #### Limitations
- * As of `v1.9.0` the module requires explicit creation of a class. If you rely on auto-schema to create the class for you, it will be missing the required configuration about which fields should be vectorized. This will be addressed in a future release.
+:::
-* *Fixes*
- * fix an error where deleting a class with `geoCoordinates` could lead to a panic due to missing cleanup (#1730)
- * fix an issue where an error in a module would not be forwarded to the user (#1754)
- * fix an issue where a class could not be deleted on some file system (e.g. AWS EFS) (#1757)
+### Raft Migration (v1.25.0+)
+Weaviate `v1.25.0` introduced Raft [as the consensus algorithm for cluster metadata](../../concepts/replication-architecture/cluster-architecture.md#metadata-replication-raft). This requires a one-time migration of the cluster metadata.
-## Migration to version v1.8.0
+In [Docker-based self-hosted instances](../../installation/docker-compose.md), the migration is automatic.
-### Migration Notice
+In [Kubernetes-based self-hosted instances](../../installation/kubernetes.md), you must perform a manual migration step. For more information, see the [Weaviate `v1.25.0` migration guide](./weaviate-1-25.md).
-Version `v1.8.0` introduces multi-shard indexes and horizontal scaling. As a
-result the dataset needs to be migrated. This migration is performed automatically -
-without user interaction - when first starting up with Weaviate version
-`v1.8.0`. However, it cannot be reversed. We, therefore, recommend carefully
-reading the following migration notes and making a case-by-case decision about the
-best upgrade path for your needs.
+This was a significant change to the Weaviate architecture. Accordingly, we suggest performing another backup after upgrading to `v1.25.latest`, before proceeding with further upgrades to ensure that you have a recent backup.
-#### Why is a data migration necessary?
+### Backup Restoration Fix (v1.23.13+)
-Prior to `v1.8.0` Weaviate did not support multi-shard indexes. The feature was
-already planned, therefore data was already contained in a single shard with a
-fixed name. A migration is necessary to move the data from a single fixed shard
-into a multi-shard setup. The amount of shards is not changed. When you run
-`v1.8.0` on a dataset the following steps happen automatically:
-
-* Weaviate discovers the missing sharding configuration for your classes and fills it with the default values
-* When shards start-up and they do not exist on disk, but a shard with a fixed
- name from `v1.7.x` exists, Weaviate automatically recognizes that a migration
- is necessary and moves the data on disk
-* When Weaviate is up and running the data has been migrated.
-
-**Important Notice:** As part of the migration Weaviate will assign the shard
-to the (only) node available in the cluster. You need to make sure that this
-node has a stable hostname. If you run on Kubernetes, hostnames are stable
-(e.g. `weaviate-0` for the first node). However with `Docker Compose` hostnames
-default to the id of the container. If you remove your containers (e.g.
-`docker compose down`) and start them up again, the hostname will have changed.
-This will lead to errors where Weaviate mentions that it cannot find the node
-that the shard belongs to. The node sending the error message is the node that
-owns the shard itself, but it cannot recognize it, since its own name has
-changed.
-
-To remedy this, you can set a stable hostname **before starting up with
-v1.8.0** by setting the env var `CLUSTER_HOSTNAME=node1`. The actual name does
-not matter, as long as it's stable.
-
-If you forgot to set a stable hostname and are now running into the error
-mentioned above, you can still explicitly set the hostname that was used before
-which you can derive from the error message.
-
-Example:
-
-If you see the error message `"shard Knuw6a360eCY: resolve node name
-\"5b6030dbf9ea\" to host"`, you can make Weaviate usable again, by setting
-`5b6030dbf9ea` as the host name: `CLUSTER_HOSTNAME=5b6030dbf9ea`.
-
-#### Should you upgrade or reimport?
-
-In addition to new features, `v1.8.0` also contains a large collection of
-bugfixes. Some of the bugs affect how Weaviate writes the HNSW index to disk.
-A pre-1.18.0 index on disk may not be as good a freshly built `v1.8.0` index.
-If you can import using a script, we generally recommend starting with a fresh
-`v1.8.0` instance and reimporting instead of migrating.
-
-#### Is downgrading possible after upgrading?
-
-Note that the data migration which happens at the first startup of v1.8.0 is
-not automatically reversible. If you plan on downgrading to `v1.7.x` again
-after upgrading, you must explicitly create a backup of the state prior to
-upgrading.
-
-### Changelog
-
-
-## Changelog for version v1.7.2
-* No breaking changes
-* New features
- * ### Array Datatypes (#1691)
- Added `boolean[]` and `date[]`.
- * ### Make property names less strict (#1562)
- Property names in a data schema allows: `/[_A-Za-z][_0-9A-Za-z]*/`. i.e. it will allow for using underscores, will allow numbers and will fix the issue about trailing upper-case characters. But it won't allow for many other special characters, such as dash (-) or language-specific characters like Umlauts, etc, due to GraphQL restrictions.
-* Bug fixes
- * ### Aggregation on array data type (#1686)
-
-
-## Changelog for version v1.7.0
-* No breaking changes
-* New features
- * ### Array Datatypes (#1611)
- Starting with this release, primitive object properties are no longer limited to individual properties, but can also include lists of primitives. Array types can be stored, filtered and aggregated in the same way as other primitives.
-
- Auto-schema will automatically recognize lists of `string`/`text` and `number`/`int`. You can also explicitly specify lists in the schema by using the following data types `string[]`, `text[]`, `int[]`, `number[]`. A type that is assigned to be an array, must always stay an array, even if it only contains a single element.
-
- * ### New Module: `text-spellcheck` - Check and autocorrect misspelled search terms (#1606)
- Use the new spellchecker module to verify user-provided search queries (in existing `nearText` or `ask` functions) are spelled correctly and even suggest alternative, correct spellings. Spell-checking happens at query time.
-
- There are two ways to use this module:
- 1. It provides a new additional property which can be used to check (but not alter) the provided queries:
- The following query:
- ```graphql
- {
- Get {
- Post(nearText: {
- concepts: "missspelled text"
- }) {
- content
- _additional {
- spellCheck {
- changes {
- corrected
- original
- }
- didYouMean
- location
- originalText
- }
- }
- }
- }
- }
- ```
-
- will produce results similar to the following:
-
- ```
- "_additional": {
- "spellCheck": [
- {
- "changes": [
- {
- "corrected": "misspelled",
- "original": "missspelled"
- }
- ],
- "didYouMean": "misspelled text",
- "location": "nearText.concepts[0]",
- "originalText": "missspelled text"
- }
- ]
- },
- "content": "..."
- },
- ```
- 2. It extends existing `text2vec-*` modules with an `autoCorrect` flag, which can be used to automatically correct the query if misspelled.
-
- * ### New Module `ner-transformers` - Extract entities from Weaviate using transformers (#1632)
- Use transformer-based models to extract entities from your existing Weaviate objects on the fly. Entity extraction happens at query time. Note that for maximum performance, transformer-based models should run with GPUs. CPUs can be used, but the throughput will be lower.
-
- To make use of the module's capabilities, simply extend your query with the following new `_additional` property:
-
- ```graphql
- {
- Get {
- Post {
- content
- _additional {
- tokens(
- properties: ["content"], # is required
- limit: 10, # optional, int
- certainty: 0.8 # optional, float
- ) {
- certainty
- endPosition
- entity
- property
- startPosition
- word
- }
- }
- }
- }
- }
-
- ```
- It will return results similar to the following:
-
- ```
- "_additional": {
- "tokens": [
- {
- "property": "content",
- "entity": "PER",
- "certainty": 0.9894614815711975,
- "word": "Sarah",
- "startPosition": 11,
- "endPosition": 16
- },
- {
- "property": "content",
- "entity": "LOC",
- "certainty": 0.7529033422470093,
- "word": "London",
- "startPosition": 31,
- "endPosition": 37
- }
- ]
- }
- ```
-* Bug fixes
- * Aggregation can get stuck when aggregating `number` datatypes (#1660)
-
-## Changelog for version 1.6.0
-* No breaking changes
-* No new features
- * **Zero Shot Classification (#1603)** This release adds a new classification type `zeroshot` that works with any `vectorizer` or custom vectors. It picks the label objects that have the lowest distance to the source objects. The link is made using cross-references, similar to existing classifications in Weaviate. To start a `zeroshot` classification use `"type": "zeroshot"` in your `POST /v1/classficiations` request and specify the properties you want classified normally using `"classifyProperties": [...]`. As zero shot involves no training data, you cannot set `trainingSetWhere` filters, but can filter both source (`"sourceWhere"`) and label objects (`"targetWhere"`) directly.
-* Bug fixes
-
-
-## Changelog for version 1.5.2
-
-* No breaking changes
-* No new features
-* Bug fixes:
-* ### Fix possible data races (`short write`) (#1643)
- This release fixes various possible data races that could in the worst case lead to an unrecoverable error `"short write"`. The possibility for those races was introduced in `v.1.5.0` and we highly recommend anyone running on the `v1.5.x` timeline to upgrade to `v1.5.2` immediately.
-
-## Changelog for version 1.5.1
-
-* No breaking changes
-* No new features
-* Bug fixes:
-* ### Crashloop after unexpected crash in HNSW commit log (#1635)
- If Weaviate was killed (e.g. OOMKill) while writing the commit log, it could not be parsed after the next restart anymore, thus ending up in a crashloop. This fix removes this. Note that no data will be lost on such a crash: The partially written commit log has not yet been acknowledged to the user, so no write guarantees have been given yet. It is therefore safe to discard.
-
-* ### Chained Like operator not working (#1638)
- Prior to this fix, when chaining `Like` operators in `where` filters where each `valueString` or `valueText` contained a wildcard (`*`), typically only the first operator's results where reflected. This fix makes sure that the chaining (`And` or `Or`) is reflected correctly. This bug did not affect other operators (e.g. `Equal`, `GreaterThan`, etc) and only affected those `Like` queries where a wildcard was used.
-
-* ### Fix potential data race in Auto Schema features (#1636)
- This fix improves incorrect synchronization on the auto schema feature which in extreme cases could lead to a data race.
-
-## Migration to version 1.5.0
-
-### Migration Notice
-*This release does not contain any API-level breaking changes, however, it changes the entire storage mechanism inside Weaviate. As a result, an in-place update is not possible. When upgrading from previous versions, a new setup needs to be created and all data reimported. Prior backups are not compatible with this version.*
-
-### Changelog
-* No breaking changes
-* New Features:
- * *LSM-Tree based Storage*. Previous releases of Weaviate used a B+Tree based storage mechanism. This was not fast enough to keep up with the high write speed requirements of a large-scale import. This release completely rewrites the storage layer of Weaviate to use a custom LSM-tree approach. This leads to considerably faster import times, often more than 100% faster than the previous version.
- * *Auto-Schema Feature*. Import data objects without creating a schema prior to import. The classes will be created automatically, they can still be adjusted manually. Weaviate will guess the property type based on the first time it sees a property. The defaults can be configured using the environment variables outlined in #1539. The feature is on by default, but entirely non-breaking. You can still create an explicit schema at will.
-* Fixes:
- * *Improve Aggregation Queries*. Reduces the amount of allocations required for some aggregation queries, speeding them up and reduces the amount of timeouts encountered during aggregations.
-
-
-Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.5.0) for all the changes.
-
-
-## Changelog for version 1.4.0
-
-* No breaking changes
-* New Features:
- * Image Module [`img2vec-neural`](/developers/weaviate/modules/img2vec-neural.md)
- * Add Hardware acceleration for `amd64` CPUs (Intel, AMD)
- * Support `arm64` technology for entire Weaviate stack
- * Set `ef` at search time
- * Introduce new dataType `blob`
- * Skip vector-indexing a class
-* Fixes:
- * Various Performance Fixes around the HNSW Vector Index
- * Make property order consistent when vectorizing
- * Fix issues around `PATCH` API when using custom vectors
- * Detect schema settings that will most likely lead to duplicate vectors and print warning
- * Fix missing schema validation on transformers module
-
-Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.4.0) for all the changes.
-
-
-## Changelog for version 1.3.0
-
-* No breaking changes
-* New feature: [Question Answering (Q&A) Module](/developers/weaviate/modules/qna-transformers.md)
-* New feature: New Meta Information for all transformer-based modules
-
-Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.3.0) for all the changes.
-
-## Changelog for version 1.2.0
-
-* No breaking changes
-* New feature: Introduction of the [Transformer Module](/developers/weaviate/modules/qna-transformers.md)
-
-Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.2.0) for all the changes.
-
-## Changelog for version 1.1.0
-
-* No breaking changes
-* New feature: GraphQL `nearObject` search to get most similar object.
-* Architectural update: Cross-reference batch import speed improvements.
-
-Check [this github page](https://github.com/weaviate/weaviate/releases/tag/v1.1.0) for all the changes.
-
-## Migration to version 1.0.0
-
-Weaviate version 1.0.0 was released on 12 January 2021, and consists of the major update of modularization. From version 1.0.0, Weaviate is modular, meaning that the underlying structure relies on a *pluggable* vector index, *pluggable* vectorization modules with possibility to extend with *custom* modules.
-
-Weaviate release 1.0.0 from 0.23.2 comes with a significant amount of breaking changes in the data schema, API and clients. Here is an overview of all (breaking) changes.
-
-For client library specific changes, take a look at the change logs of the specific client ([Go](/developers/weaviate/client-libraries/go.md#change-logs), [Python](/developers/weaviate/client-libraries/python/index.md#change-logs) and [TypeScript/JavaScript](/developers/weaviate/client-libraries/typescript/index.mdx#changelog).
-
-Moreover, a new version of the Console is released. Visit the Console documentation for more information.
-
-### Summary
-This contains most overall changes, but not all details. Those are documented in ["Changes"](#changes).
-
-#### All RESTful API changes
-* from `/v1/schema/things/{ClassName}` to `/v1/schema/{ClassName}`
-* from `/v1/schema/actions/{ClassName} `to `/v1/schema/{ClassName}`from `/v1/schema/actions/{ClassName} `to `/v1/schema/{ClassName}`
-* from `/v1/things` to `/v1/objects`
-* from `/v1/actions` to `/v1/objects`
-* from `/v1/batching/things` to `/v1/batch/objects`
-* from `/v1/batching/actions` to `/v1/batch/objects`
-* from `/v1/batching/references` to `/v1/batch/references`
-* Additional data object properties are grouped in `?include=...` and the leading underscore of these properties is removed
-* The `/v1/modules/` endpoint is introduced.
-* the `/v1/meta/` endpoint now contains module specific information in `"modules"`
-
-#### All GraphQL API changes
-* Removal of Things and Actions layer in query hierarchy
-* Reference properties of data objects are lowercase (previously uppercased)
-* Underscore properties, uuid and certainty are now grouped in the object `_additional`
-* `explore()` filter is renamed to `near` filter
-* `nearVector(vector:[])` filter is introduced in `Get{}` query
-* `Explore (concepts: ["foo"]){}` query is changed to `Explore (near: ... ) {}`.
-
-#### All data schema changes
-* Removal of Things and Actions
-* Per class and per property configuration is changed to support modules and vector index type settings.
-
-#### All data object changes
-* From `schema` to `properties` in the data object.
-
-#### Contextionary
-* Contextionary is renamed to the module `text2vec-contextionary`
-* `/v1/c11y/concepts` to `/v1/modules/text2vec-contextionary/concepts`
-* `/v1/c11y/extensions` to `/v1/modules/text2vec-contextionary/extensions`
-* `/v1/c11y/corpus` is removed
-
-#### Other
-* Removal of `/things` and `/actions` in short and long beacons
-* Classification body is changed to support modularization
-* `DEFAULT_VECTORIZER_MODULE` is a new environment variable
-
-### Changes
-
-#### Removal of Things and Actions
-`Things` and `Actions` are removed from the Data Schema. This comes with the following changes in the schema definition and API endpoints:
-1. **Data schema:** The `semantic kind` (`Things` and `Actions`) is removed from the Schema Endpoint. This means the URLs will change:
- * from `/v1/schema/things/{ClassName}` to `/v1/schema/{ClassName}`
- * from `/v1/schema/actions/{ClassName} `to `/v1/schema/{ClassName}`
-1. **Data RESTful API endpoint:** The `semantic kind` (`Things` and `Actions`) is removed from the data Endpoint. Instead it will be namespaced as `/objects`. This means the URLs will change:
- * from `/v1/things` to `/v1/objects`
- * from `/v1/actions` to `/v1/objects`
- * from `/v1/batching/things`to `/v1/batch/objects` (see also the [change in batching](#renaming-batching-to-batch))
- * from `/v1/batching/actions`to `/v1/batch/objects` (see also the [change in batching](#renaming-batching-to-batch))
-1. **GraphQL:** The `Semantic Kind` "level" in the query hierarchy will be removed without replacement (In `Get` and `Aggregate` queries), i.e.
- ```graphql
- {
- Get {
- Things {
- ClassName {
- propName
- }
- }
- }
- }
- ```
-
- will become
-
- ```graphql
- {
- Get {
- ClassName {
- propName
- }
- }
- }
- ```
-1. **Data Beacons:** The `Semantic Kind` will be removed from beacons:
- * **Short-form Beacon:**
-
- * `weaviate://localhost/things/4fbacd6e-1153-47b1-8cb5-f787a7f01718`
-
- to
-
- * `weaviate://localhost/4fbacd6e-1153-47b1-8cb5-f787a7f01718`
-
- * **Long-form Beacon:**
-
- * `weaviate://localhost/things/ClassName/4fbacd6e-1153-47b1-8cb5-f787a7f01718/propName`
-
- to
-
- * `weaviate://localhost/ClassName/4fbacd6e-1153-47b1-8cb5-f787a7f01718/propName`
-
-#### Renaming /batching/ to /batch/
-
-* `/v1/batching/things` to `/v1/batch/objects`
-* `/v1/batching/actions` to `/v1/batch/objects`
-* `/v1/batching/references` to `/v1/batch/references`
-
-
-#### From "schema" to "properties" in data object
-
-The name "schema" on the data object is not intuitive and is replaced by "properties". The change looks like:
-
-```json
-{
- "class": "Article",
- "schema": {
- "author": "Jane Doe"
- }
-}
-```
-
-to
-
-```json
-{
- "class": "Article",
- "properties": {
- "author": "Jane Doe"
- }
-}
-```
-
-#### Consistent casing in GraphQL properties
-
-Previously, reference properties in the schema definitions are always lowercase, yet in graphQL they needed to be uppercased. E.g.: `Article { OfAuthor { … on Author { name } } } }`, even though the property is defined as ofAuthor. New is that the casing in GraphQL reflects exactly the casing in the schema definition, thus the above example would become: `Article { ofAuthor { … on Author { name } } } }`
-
-#### Additional data properties in GraphQL and RESTful API
-Since modularization, a module can contribute to the additional properties of a data object (thus are not fixed), which should be retrievable by the GraphQL and/or RESTful API.
-1. **REST**: `additional` properties (formerly named `"underscore"` properties) can be included in RESTful query calls like `?include=...`, e.g. `?include=classification`. The underscores will thus be removed from the names (e.g. `?include=_classification` is deprecated). In the Open API specifications, all additional properties will be grouped in the object `additional`. For example:
- ```json
- {
- "class": "Article",
- "schema": { ... },
- "_classification": { … }
- }
- ```
-
- to
-
- ```json
- {
- "class": "Article",
- "properties": { ... },
- "additional": {
- "classification": { ... }
- }
- }
- ```
-2. **GraphQL**: `"underscore"` properties are renamed to `additional` properties in GraphQL queries.
- 1. All former `"underscore"` properties of a data object (e.g. `_certainty`) are now grouped in the `_additional {}` object (e.g. `_additional { certainty } `).
- 2. The `uuid` property is now also placed in the `_additional {}` object and renamed to `id` (e.g. `_additional { id } `).
- This example covers both changes:
-
- From
-
- ```graphql
- {
- Get {
- Things {
- Article {
- title
- uuid
- certainty
- _classification
- }
- }
- }
- }
- ```
-
- to
-
- ```graphql
- {
- Get {
- Article {
- title
- _additional { # leading _ prevents clashes
- certainty
- id # replaces uuid
- classification
- }
- }
- }
- }
- ```
-
-#### Modules RESTful endpoint
-With the modularization of Weaviate, the `v1/modules/` endpoint is introduced.
-
-#### GraphQL semantic search
-
-With the modularization, it becomes possible to vectorize non-text objects. Search is no longer restricted to use the Contextionary's vectorization of text and data objects, but could also be applied to non-text objects or raw vectors. The formerly 'explore' filter in Get queries and 'Explore' queries in GraphQL were tied to text, but the following changes are made to this filter with the new version of Weaviate:
-
-1. The filter `Get ( explore: {} ) {}` is renamed to `Get ( near: {} ) {}`.
- 1. New: `Get ( nearVector: { vector: [.., .., ..] } ) {}` is module independent and will thus always be available.
- 2. `Get ( explore { concepts: ["foo"] } ) {}` will become `Get ( nearText: { concepts: ["foo"] } ) {}` and is only available if the `text2vec-contextionary` module is attached.
-
- From
-
- ```graphql
- {
- Get {
- Things {
- Article (explore: { concepts: ["foo"] } ) {
- title
- }
- }
- }
- }
- ```
-
- to
-
- ```graphql
- {
- Get {
- Article (near: ... ) {
- title
- }
- }
- }
- ```
-
-2. Similarly to the explore sorter that is used in the `Get {}` API, the `Explore {}` API also assumes text. The following change is applied:
-
- From
-
- ```graphql
- {
- Explore (concepts: ["foo"]) {
- beacon
- }
- }
- ```
-
- to
-
- ```graphql
- {
- Explore (near: ... ) {
- beacon
- }
- }
- ```
-
-#### Data schema configuration
-1. **Per-class configuration**
-
- With modularization, it is possible to configure per class the vectorizer module, module-specific configuration for the overall class, vector index type, and vector index type specific configuration:
- * The `vectorizer` indicates which module (if any) are responsible for vectorization.
- * The `moduleConfig` allows configuration per module (by name).
- * See [here](#text2vec-contextionary) for Contextionary specific property configuration.
- * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/vector-index.md#hnsw))
- * The `vectorIndexConfig` is an arbitrary object passed to the index for config (defaults can be found [here](/developers/weaviate/config-refs/schema/vector-index.md#how-to-configure-hnsw) )
-
- All changes are in this example:
-
- ```json
- {
- "class": "Article",
- "vectorizeClassName": true,
- "description": "string",
- "properties": [ … ]
- }
- ```
-
- will become
-
- ```json
- {
- "class": "Article",
- "vectorIndexType": "hnsw", # defaults to hnsw
- "vectorIndexConfig": {
- "efConstruction": 100
- },
- "moduleConfig": {
- "text2vec-contextionary": {
- "vectorizeClassName": true
- },
- "encryptor5000000": { "enabled": true } # example
- },
- "description": "string",
- "vectorizer": "text2vec-contextionary", # default is configurable
- "properties": [ … ]
- }
- ```
-
-2. **Per-property configuration**
-
- With modularization, it is possible to configure per property module-specific configuration per property if available and it can be specified if a property should be included in the inverted index.
- * The `moduleConfig` allows configuration per module (by name).
- * See [here](#text2vec-contextionary) for Contextionary specific property configuration.
- * `index` will become `indexInverted`: a boolean that indicates whether a property should be indexed in the inverted index.
-
- All changes are in this example:
-
- ```json
- {
- "dataType": [ "text" ],
- "description": "string",
- "cardinality": "string",
- "vectorizePropertyName": true,
- "name": "string",
- "keywords": [ … ],
- "index": true
- }
- ```
-
- will become
-
- ```json
- {
- "dataType": [ "text" ],
- "description": "string",
- "moduleConfig": {
- "text2vec-contextionary": {
- "skip": true,
- "vectorizePropertyName": true,
- }
- },
- "name": "string",
- "indexInverted": true
- }
- ```
-
-#### RESTful /meta endpoint
-
-The `/v1/meta` object now contains module specific information at in the newly introduced namespaced `modules.` property:
-
-From
-
-```json
-{
- "hostname": "string",
- "version": "string",
- "contextionaryWordCount": 0,
- "contextionaryVersion": "string"
-}
-```
-
-to
-
-```json
-{
- "hostname": "string",
- "version": "string",
- "modules": {
- "text2vec-contextionary": {
- "wordCount": 0,
- "version": "string"
- }
- }
-}
-```
-
-#### Modular classification
-
-Some classification types are tied to modules (e.g. the former "contextual" classification is tied to the `text2vec-contextionary` module. We make a distinction between fields which are always present and those which are type dependent. Additionally the API is improved by grouping `settings` and `filters` in separate properties. kNN classification is the only type of classification that is present with Weaviate Core without dependency on modules. The former "contextual" classification is tied to the `text2vec-contextionary` module, see [here](#text2vec-contextionary). An example of how the change looks like in the classification API POST body:
-
-From
-
-```json
-{
- "class": "City",
- "classifyProperties": ["inCountry"],
- "basedOnProperties": ["description"],
- "type": "knn",
- "k": 3,
- "sourceWhere": { … },
- "trainingSetWhere": { … },
- "targetWhere": { … },
-}
-
-```
-
-To
-
-```json
-{
- "class": "City",
- "classifyProperties": ["inCountry"],
- "basedOnProperties": ["description"],
- "type": "knn",
- "settings": {
- "k": 3
- },
- "filters": {
- "sourceWhere": { … },
- "trainingSetWhere": { … },
- "targetWhere": { … },
- }
-}
-```
-
-And the API GET body:
-
-From
-
-```json
-{
- "id": "ee722219-b8ec-4db1-8f8d-5150bb1a9e0c",
- "class": "City",
- "classifyProperties": ["inCountry"],
- "basedOnProperties": ["description"],
- "status": "running",
- "meta": { … },
- "type": "knn",
- "k": 3,
- "sourceWhere": { … },
- "trainingSetWhere": { … },
- "targetWhere": { … },
-}
-```
-
-To
-
-```json
-{
- "id": "ee722219-b8ec-4db1-8f8d-5150bb1a9e0c",
- "class": "City",
- "classifyProperties": ["inCountry"],
- "basedOnProperties": ["description"],
- "status": "running",
- "meta": { … },
- "type": "knn",
- "settings": {
- "k": 3
- },
- "filters": {
- "sourceWhere": { … },
- "trainingSetWhere": { … },
- "targetWhere": { … },
- }
-}
-```
-
-#### text2vec-contextionary
-The Contextionary becomes the first vectorization module of Weaviate, renamed to `text2vec-contextionary` in formal use. This brings the following changes:
-1. **RESTful** endpoint `/v1/c11y` changes to `v1/modules/text2vec-contextionary`:
- * `/v1/c11y/concepts` to `/v1/modules/text2vec-contextionary/concepts`
- * `/v1/c11y/extensions` to `/v1/modules/text2vec-contextionary/extensions`
- * `/v1/c11y/corpus` is removed
-2. **Data schema:** `text2vec-contextionary`-specific module configuration options in the schema definition
- 1. **Per-class**. `"vectorizeClassName"` indicates whether the class name should be taken into the vector calculation of data objects.
-
- ```json
- {
- "class": "Article",
- "moduleConfig": {
- "text2vec-contextionary": {
- "vectorizeClassName": true
- }
- },
- "description": "string",
- "vectorizer": "text2vec-contextionary",
- "properties": [ … ]
- }
- ```
-
- 2. **Per-property.** `skip` tells whether to skip the entire property (including value) from the vector position of the data object. `vectorizePropertyName` indicates whether the property name should be taken into the vector calculation of data objects.
-
- ```json
- {
- "dataType": [ "text" ],
- "description": "string",
- "moduleConfig": {
- "text2vec-contextionary": {
- "skip": true,
- "vectorizePropertyName": true,
- }
- },
- "name": "string",
- "indexInverted": true
- }
- ```
-3. **Contextual classification**. Contextual classification is dependent on the module `text2vec-contextionary`. It can be activated in `/v1/classifications/` the following with the classification name `text2vec-contextionary-contextual`:
-
-From
-
-```json
-{
- "class": "City",
- "classifyProperties": ["inCountry"],
- "basedOnProperties": ["description"],
- "type": "contextual",
- "informationGainCutoffPercentile": 30,
- "informationGainMaximumBoost": 3,
- "tfidfCutoffPercentile": 80,
- "minimumUsableWords": 3,
- "sourceWhere": { … },
- "trainingSetWhere": { … },
- "targetWhere": { … },
-}
-```
-
-To
-
-```json
-{
- "class": "City",
- "classifyProperties": ["inCountry"],
- "basedOnProperties": ["description"],
- "type": "text2vec-contextionary-contextual",
- "settings": {
- "informationGainCutoffPercentile": 30,
- "informationGainMaximumBoost": 3,
- "tfidfCutoffPercentile": 80,
- "minimumUsableWords": 3
- },
- "filters": {
- "sourceWhere": { … },
- "trainingSetWhere": { … },
- "targetWhere": { … }
- }
-}
-```
-
-#### Default vectorizer module
-The default vectorizer module can be specified in a new environment variable so that this doesn't have to be specified on every data class in the schema. The environment variable is `DEFAULT_VECTORIZER_MODULE`, which can be set to for example `DEFAULT_VECTORIZER_MODULE="text2vec-contextionary"`.
-
-
-### Official release notes
-Official release notes can be found on [GitHub](https://github.com/weaviate/weaviate/releases/tag/0.23.0).
+Before `v1.23.13`, there was a bug with the backup restoration process, which could lead to data not being stored correctly.
+If you are upgrading from a version before `v1.23.13`, we recommend that you:
+1. Create a backup of your current Weaviate instance.
+2. Upgrade to at least `v1.23.13` (preferably to `v1.23.16`) or higher, using the [general upgrade instructions above](#general-upgrade-instructions).
+3. Restore your backup to the upgraded instance.
## Questions and feedback
diff --git a/developers/weaviate/more-resources/migration/weaviate-1-25.md b/developers/weaviate/more-resources/migration/weaviate-1-25.md
index f263740cd4..b110c686eb 100644
--- a/developers/weaviate/more-resources/migration/weaviate-1-25.md
+++ b/developers/weaviate/more-resources/migration/weaviate-1-25.md
@@ -17,24 +17,28 @@ This migration guide assumes that you have:
## Migration overview
-Weaviate `1.25` introduces [Raft](https://raft.github.io/) as the consensus algorithm for its database schema, in order to improve its fault tolerance. This change requires a migration of the entire schema.
+Weaviate `1.25` introduces [Raft](https://raft.github.io/) as the consensus algorithm for its cluster metadata, in order to improve its fault tolerance. This change requires a migration of the entire metadata.
+
+:::tip cluster metadata and schema
+The cluster metadata was previously referred to as the `schema`. We now use the term `metadata`, and use `schema` to refer to the data model of the Weaviate instance, such as classes, properties, etc.
+:::
As a result, to migrate from a pre-`1.25` version of Weaviate to `1.25` on kubernetes, you must follow these steps:
- Delete the deployed [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
- Update the helm chart to version `17.0.0` or higher
- Re-deploy Weaviate
-- Wait for schema migration to complete
+- Wait for cluster metadata migration to complete
For more details, refer to the [upgrade instructions](#upgrade-instructions) below.
-To downgrade from `1.25` to a pre-`1.25` version, you must perform a `POST` request to the `v1/cluster/schema-v1` endpoint to downgrade the schema. Then, you must similarly delete the deployed `StatefulSet` and downgrade Weaviate to the desired version.
+To downgrade from `1.25` to a pre-`1.25` version, you must perform a `POST` request to the `v1/cluster/schema-v1` endpoint to downgrade the metadata. Then, you must similarly delete the deployed `StatefulSet` and downgrade Weaviate to the desired version.
For more details, refer to the [downgrade instructions](#downgrade-instructions) below.
:::caution Cluster downtime
-This upgrade requires a schema migration. The cluster requires some downtime for the migration. The length of the downtime depends on the size of the database.
+This upgrade requires a cluster metadata migration. The cluster requires some downtime for the migration. The length of the downtime depends on the size of the database.
We suggest performing this upgrade at a least disruptive time, or even provisioning a secondary cluster while the main cluster is being restarted.
@@ -156,11 +160,11 @@ If the number of objects under `statistics` matches the number of replicas you h
## Downgrade instructions
-If you need to downgrade from `1.25` to a pre-`1.25` version, you must perform a `POST` request to the `v1/cluster/schema-v1` (a payload is not required) to downgrade the schema.
+If you need to downgrade from `1.25` to a pre-`1.25` version, you must perform a `POST` request to the `v1/cluster/schema-v1` (a payload is not required) to downgrade the cluster metadata.
-### 1. Downgrade schema
+### 1. Downgrade cluster metadata
-Perform the following request to downgrade the schema. This will prepare the cluster for a downgrade to a pre-`1.25` version. (Remember to replace `localhost:8080` with the correct URL & port.)
+Perform the following request to downgrade the cluster metadata. This will prepare the cluster for a downgrade to a pre-`1.25` version. (Remember to replace `localhost:8080` with the correct URL & port.)
```bash
curl -X POST -s -o /dev/null -w "%{http_code}" localhost:8080/v1/cluster/schema-v1
@@ -170,7 +174,7 @@ This should return a `200` status code.
### 2. Delete StatefulSet
-After downgrading the schema, delete the existing StatefulSet. This will delete all the pods in the namespace.
+After downgrading the cluster metadata, delete the existing StatefulSet. This will delete all the pods in the namespace.
```bash
kubectl delete sts weaviate -n weaviate
diff --git a/developers/weaviate/quickstart/img/create_cluster.png b/developers/weaviate/quickstart/img/create_cluster.png
index f8a5ef1e42..53941b9939 100644
Binary files a/developers/weaviate/quickstart/img/create_cluster.png and b/developers/weaviate/quickstart/img/create_cluster.png differ
diff --git a/developers/weaviate/quickstart/img/create_sandbox.png b/developers/weaviate/quickstart/img/create_sandbox.png
index f069ab0a96..ae33257fa6 100644
Binary files a/developers/weaviate/quickstart/img/create_sandbox.png and b/developers/weaviate/quickstart/img/create_sandbox.png differ
diff --git a/developers/weaviate/quickstart/img/enable_embeddings.png b/developers/weaviate/quickstart/img/enable_embeddings.png
new file mode 100644
index 0000000000..74796a3f08
Binary files /dev/null and b/developers/weaviate/quickstart/img/enable_embeddings.png differ
diff --git a/developers/weaviate/quickstart/index.md b/developers/weaviate/quickstart/index.md
index c619ff342b..b0b15d61db 100644
--- a/developers/weaviate/quickstart/index.md
+++ b/developers/weaviate/quickstart/index.md
@@ -1,11 +1,12 @@
---
-title: Quickstart
+title: Quickstart (with cloud resources)
sidebar_position: 0
image: og/docs/quickstart-tutorial.jpg
# tags: ['getting started']
hide_table_of_contents: true
---
+# Quickstart: with cloud resources
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
@@ -15,32 +16,71 @@ import TabItem from '@theme/TabItem';
:::info What you will learn
-In this quickstart guide, you will:
+This quickstart shows you how to combine Weaviate Cloud and Cohere to:
-1. Set up Weaviate. (10 minutes)
-1. Populate the database. (10 minutes)
+1. Set up a Weaviate instance. (10 minutes)
+1. Add and vectorize your data. (10 minutes)
1. Perform a semantic search and retrieval augmented generation (RAG). (10 minutes)
+```mermaid
+flowchart LR
+ %% Define nodes with white backgrounds and darker borders
+ A1["Create Weaviate\nSandbox"] --> A2["Install client\nlibrary"]
+ A2 --> A3["Connect to\nWeaviate"]
+ A3 --> B1["Define collection\n(with an inference API)"]
+ B1 --> B2["Batch import\nobjects"]
+ B2 --> C1["Semantic search\n(nearText)"]
+ C1 --> C2["RAG\n(Generate)"]
+
+ %% Group nodes in subgraphs with brand colors
+ subgraph sg1 ["1. Setup"]
+ A1
+ A2
+ A3
+ end
+
+ subgraph sg2 ["2. Populate"]
+ B1
+ B2
+ end
+
+ subgraph sg3 ["3. Query"]
+ C1
+ C2
+ end
+
+ %% Style nodes with white background and darker borders
+ style A1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style A2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style A3 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style B1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style B2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style C1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style C2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+ %% Style subgraphs with brand colors
+ style sg1 fill:#ffffff,stroke:#61BD73,stroke-width:2px,color:#130C49
+ style sg2 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+ style sg3 fill:#ffffff,stroke:#7AD6EB,stroke-width:2px,color:#130C49
+```
+
+Notes:
+
+- The code examples here are self-contained. You can copy and paste them into your own environment to try them out.
+
+- If you prefer to use locally hosted resources, see [Quickstart: locally hosted](./local.md).
:::
### Prerequisites
-This tutorial uses a Sandbox instance on [Weaviate Cloud](https://console.weaviate.cloud), and an [OpenAI](https://platform.openai.com/) API key.
+You will need accounts with [Weaviate Cloud](https://console.weaviate.cloud) and [Cohere](https://dashboard.cohere.com/). We will use free tiers (Weaviate Sandbox and Cohere trial API key) of both services.
-The Weaviate Sandbox is free, but the OpenAI usage may incur a small cost (e.g. < 10c US). If you have another, preferred [model provider](../model-providers/index.md), you can use that instead.
-
-:::note For Python users
-We have ([a Jupyter notebook](https://github.com/weaviate-tutorials/quickstart/blob/main/quickstart_end_to_end.ipynb)) available, or you can try it on [Google Colab](https://colab.research.google.com/github/weaviate-tutorials/quickstart/blob/main/quickstart_end_to_end.ipynb).
-:::
+If you have another, preferred [model provider](../model-providers/index.md), you can use that instead of Cohere.
-:::tip Try it yourself
-The code examples here are self-contained. You can copy and paste them into your own environment to try them out.
-:::
-
## Step 1: Set up Weaviate
### 1.1 Create a Weaviate database
@@ -196,8 +236,8 @@ A collection is a set of objects that share the same data structure, like a tabl
:::
The following example creates a *collection* called `Question` with:
- - OpenAI [embedding model integration](../model-providers/openai/embeddings.md) to create vectors during ingestion & queries.
- - OpenAI [generative AI integrations](../model-providers/openai/generative.md) for retrieval augmented generation (RAG).
+ - Cohere [embedding model integration](../model-providers/cohere/embeddings.md) to create vectors during ingestion & queries.
+ - Cohere [generative AI integrations](../model-providers/cohere/generative.md) for retrieval augmented generation (RAG).
import CreateCollection from '/_includes/code/quickstart/quickstart.create_collection.mdx'
@@ -205,34 +245,36 @@ import CreateCollection from '/_includes/code/quickstart/quickstart.create_colle
Run this code to create the collection to which you can add data.
+:::info What models are being used?
+You can optionally specify the model in the collection definition. As we did not specify models in the collection definition above, these integrations will use the Weaviate-defined default models.
+
+
+See the [model providers integration](../model-providers/index.md) section for more information.
+:::
+
Do you prefer a different setup?
Weaviate is very flexible. If you prefer a different model provider integration, or prefer to import your own vectors, see one of the following guides:
-
-
-
-
-
-
Prefer a different model provider?
-
-
- See this section for information on how to user another provider, such as AWS, Cohere, Google, and many more.
-
+
+
+
+
Prefer a different model provider?
+
+
+ See this section for information on how to user another provider, such as AWS, Cohere, Google, and many more.
@@ -258,8 +300,8 @@ import ImportObjects from '/_includes/code/quickstart/quickstart.import_objects.
Run this code to add the demo data.
-:::info OpenAI API key in the header
-Note that this code includes an additional header for the OpenAI API key. Weaviate uses this key to generate vector embeddings for the data objects as they are being added.
+:::info Cohere API key in the header
+Note that this code includes an additional header for the Cohere API key. Weaviate uses this key to generate vector embeddings for the data objects as they are being added.
:::
@@ -304,8 +346,39 @@ If you inspect the full response, you will see that the word `biology` does not
Even so, Weaviate was able to return biology-related entries. This is made possible by *vector embeddings* that capture meaning. Under the hood, semantic search is powered by vectors, or vector embeddings.
+Here is a diagram showing the workflow in Weaviate.
+
+```mermaid
+flowchart LR
+ Query["🔍 Search:\n'biology'"]
+
+ subgraph sg1 ["Vector Search"]
+ direction LR
+ VS1["Convert query\nto vector"] --> VS2["Find similar\nvectors"]
+ VS2 --> VS3["Return top\nmatches"]
+ end
+
+ subgraph sg2 ["Results"]
+ R1["Most similar\ndocuments"]
+ end
+
+ Query --> VS1
+ VS3 --> R1
+
+ %% Style nodes with white background and darker borders
+ style Query fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS3 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style R1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+ %% Style subgraphs with brand colors
+ style sg1 fill:#ffffff,stroke:#61BD73,stroke-width:2px,color:#130C49
+ style sg2 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+```
+
:::info Where did the vectors come from?
-Weaviate used the OpenAI API key to generate a vector embedding for each object during import. During the query, Weaviate similarly converted the query (`biology`) into a vector.
+Weaviate used the Cohere API key to generate a vector embedding for each object during import. During the query, Weaviate similarly converted the query (`biology`) into a vector.
As we mentioned above, this is optional. See [Starter Guide: Bring Your Own Vectors](/developers/weaviate/starter-guides/custom-vectors.mdx) if you would prefer to provide your own vectors.
:::
@@ -320,7 +393,59 @@ Weaviate is capable of many types of searches. See, for example, our how-to guid
Retrieval augmented generation (RAG), also called generative search, combines the power of generative AI models such as large language models (LLMs) with the up-to-date truthfulness of a database.
-RAG work by prompting a large language model (LLM) with a combination of a *user query* and *data retrieved from a database*.
+RAG works by prompting a large language model (LLM) with a combination of a *user query* and *data retrieved from a database*.
+
+This diagram shows the RAG workflow in Weaviate.
+
+```mermaid
+flowchart LR
+ subgraph sg0 ["Weaviate Query"]
+ direction TB
+ Search["🔍 Search: \n'biology'"]
+ Prompt["✍️ Prompt: \n'Write a\ntweet...'"]
+ end
+
+ subgraph sg1 ["Vector Search"]
+ direction LR
+ VS1["Convert query\nto vector"] --> VS2["Find similar\nvectors"]
+ VS2 --> VS3["Return top\nmatches"]
+ end
+
+ subgraph sg2 ["Generation"]
+ direction LR
+ G1["Send\n(results + prompt)\nto LLM"]
+ G1 --> G2["Generate\nresponse"]
+ end
+
+ subgraph sg3 ["Results"]
+ direction TB
+ R1["Most similar\ndocuments"]
+ R2["Generated\ncontent"]
+ end
+
+ Search --> VS1
+ VS3 --> R1
+ Prompt --> G1
+ VS3 --> G1
+ G2 --> R2
+
+ %% Style nodes with white background and darker borders
+ style Search fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style Prompt fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS3 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style G1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style G2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style R1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style R2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+ %% Style subgraphs with brand colors
+ style sg0 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+ style sg1 fill:#ffffff,stroke:#61BD73,stroke-width:2px,color:#130C49
+ style sg2 fill:#ffffff,stroke:#7AD6EB,stroke-width:2px,color:#130C49
+ style sg3 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+```
The following example combines the same search (for `biology`) with a prompt to generate a tweet.
@@ -369,7 +494,7 @@ Try these additional resources to learn more about Weaviate:
@@ -426,7 +551,7 @@ We provide answers to some common questions, or potential issues below.
See answer
-In this example, we use the `OpenAI` inference API. But you can use others.
+In this example, we use the `Cohere` inference API. But you can use others.
If you do want to change the embeddings, or the generative AI integrations, you can. You will need to:
- Ensure that the Weaviate module is available in the Weaviate instance you are using,
@@ -473,7 +598,7 @@ You should see:
{
"class": "Question",
... // truncated additional information here
- "vectorizer": "text2vec-openai"
+ "vectorizer": "text2vec-cohere"
}
]
}
diff --git a/developers/weaviate/quickstart/local.md b/developers/weaviate/quickstart/local.md
new file mode 100644
index 0000000000..1a2846c594
--- /dev/null
+++ b/developers/weaviate/quickstart/local.md
@@ -0,0 +1,578 @@
+---
+title: Locally hosted
+sidebar_position: 0
+image: og/docs/quickstart-tutorial.jpg
+# tags: ['getting started']
+hide_table_of_contents: true
+---
+
+# Quickstart: locally hosted
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+Expected time: 30 minutesPrerequisites: None
+
+
+:::info What you will learn
+
+This quickstart shows you how to combine open-source Weaviate and Ollama to:
+
+1. Set up a Weaviate instance. (10 minutes)
+1. Add and vectorize your data. (10 minutes)
+1. Perform a semantic search and retrieval augmented generation (RAG). (10 minutes)
+
+```mermaid
+flowchart LR
+ %% Define nodes with white backgrounds and darker borders
+ A1["Run Weaviate\nwith Docker"] --> A2["Install client\nlibrary"]
+ A2 --> A3["Connect to\nWeaviate"]
+ A3 --> B1["Define collection\n(with Ollama)"]
+ B1 --> B2["Batch import\nobjects"]
+ B2 --> C1["Semantic search\n(nearText)"]
+ C1 --> C2["RAG\n(Generate)"]
+
+ %% Group nodes in subgraphs with brand colors
+ subgraph sg1 ["1. Setup"]
+ A1
+ A2
+ A3
+ end
+
+ subgraph sg2 ["2. Populate"]
+ B1
+ B2
+ end
+
+ subgraph sg3 ["3. Query"]
+ C1
+ C2
+ end
+
+ %% Style nodes with white background and darker borders
+ style A1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style A2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style A3 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style B1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style B2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style C1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style C2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+ %% Style subgraphs with brand colors
+ style sg1 fill:#ffffff,stroke:#61BD73,stroke-width:2px,color:#130C49
+ style sg2 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+ style sg3 fill:#ffffff,stroke:#7AD6EB,stroke-width:2px,color:#130C49
+```
+
+Notes:
+
+- The code examples here are self-contained. You can copy and paste them into your own environment to try them out.
+
+- If you prefer to use cloud-based resources, see [Quickstart: with cloud resources](./local.md).
+:::
+
+
+
+### Prerequisites
+
+Before we get started, install [Docker](https://docs.docker.com/get-started/get-docker/) and [Ollama](https://ollama.com/download) on your machine.
+
+Then, download the `nomic-embed-text` and `llama3.2` models by running the following command:
+
+```bash
+ollama pull nomic-embed-text
+ollama pull llama3.2
+```
+
+We will be running Weaviate and language models locally. We recommend that you use a modern computer with at least 8GB or RAM, preferably 16GB or more.
+
+
+
+## Step 1: Set up Weaviate
+
+### 1.1 Create a Weaviate database
+
+Save the following code to a file named `docker-compose.yml` in your project directory.
+
+```yaml
+---
+services:
+ weaviate:
+ command:
+ - --host
+ - 0.0.0.0
+ - --port
+ - '8080'
+ - --scheme
+ - http
+ image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version||
+ ports:
+ - 8080:8080
+ - 50051:50051
+ volumes:
+ - weaviate_data:/var/lib/weaviate
+ restart: on-failure:0
+ environment:
+ QUERY_DEFAULTS_LIMIT: 25
+ AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+ PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+ ENABLE_API_BASED_MODULES: 'true'
+ ENABLE_MODULES: 'text2vec-ollama,generative-ollama'
+ CLUSTER_HOSTNAME: 'node1'
+volumes:
+ weaviate_data:
+...
+```
+
+Run the following command to start a Weaviate instance using Docker:
+
+```bash
+docker-compose up -d
+```
+
+### 1.2 Install a client library
+
+We recommend using a [client library](../client-libraries/index.md) to work with Weaviate. Follow the instructions below to install one of the official client libraries, available in [Python](../client-libraries/python/index.md), [JavaScript/TypeScript](../client-libraries/typescript/index.mdx), [Go](../client-libraries/go.md), and [Java](../client-libraries/java.md).
+
+import CodeClientInstall from '/_includes/code/quickstart/clients.install.mdx';
+
+
+
+### 1.3: Connect to Weaviate
+
+Now you can connect to your Weaviate instance.
+
+The example below shows how to connect to Weaviate and perform a basic operation, like checking the cluster status.
+
+import ConnectIsReady from '/_includes/code/quickstart/local.quickstart.is_ready.mdx'
+
+
+
+If you did not see any errors, you are ready to proceed. We will replace the simple cluster status check with more meaningful operations in the next steps.
+
+
+
+## Step 2: Populate the database
+
+Now, we can populate our database by first defining a collection then adding data.
+
+### 2.1 Define a collection
+
+:::info What is a collection?
+A collection is a set of objects that share the same data structure, like a table in relational databases or a collection in NoSQL databases. A collection also includes additional configurations that define how the data objects are stored and indexed.
+:::
+
+The following example creates a *collection* called `Question` with:
+ - Ollama [embedding model integration](../model-providers/ollama/embeddings.md) to create vectors during ingestion & queries, using the `nomic-embed-text` model, and
+ - Ollama [generative AI integrations](../model-providers/ollama/generative.md) for retrieval augmented generation (RAG), using the `llama3.2` model.
+
+import CreateCollection from '/_includes/code/quickstart/local.quickstart.create_collection.mdx'
+
+
+
+Run this code to create the collection to which you can add data.
+
+
+ Do you prefer a different setup?
+
+Weaviate is very flexible. If you prefer a different model provider integration, or prefer to import your own vectors, see one of the following guides:
+
+
+
+
+
+
Prefer a different model provider?
+
+
+ See this section for information on how to user another provider, such as AWS, Cohere, Google, and many more.
+
+
+
+
+### 2.2 Add objects
+
+We can now add data to our collection.
+
+The following example:
+- Loads objects, and
+- Adds objects to the target collection (`Question`) using a batch process.
+
+:::tip Batch imports
+([Batch imports](../manage-data/import.mdx)) are the most efficient way to add large amounts of data, as it sends multiple objects in a single request. See the [How-to: Batch import](../manage-data/import.mdx) guide for more information.
+:::
+
+import ImportObjects from '/_includes/code/quickstart/local.quickstart.import_objects.mdx'
+
+
+
+Run this code to add the demo data.
+
+
+
+## Step 3: Queries
+
+Weaviate provides a wide range of query tools to help you find the right data. We will try a few searches here.
+
+### 3.1 Semantic search
+
+Semantic search finds results based on meaning. This is called `nearText` in Weaviate.
+
+The following example searches for 2 objects whose meaning is most similar to that of `biology`.
+
+import QueryNearText from '/_includes/code/quickstart/local.quickstart.query.neartext.mdx'
+
+
+
+Run this code to perform the query. Our query found entries for `DNA` and `species`.
+
+
+ Example full response in JSON format
+
+```json
+{
+ {
+ "answer": "DNA",
+ "question": "In 1953 Watson & Crick built a model of the molecular structure of this, the gene-carrying substance",
+ "category": "SCIENCE"
+ },
+ {
+ "answer": "species",
+ "question": "2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new one of this classification",
+ "category": "SCIENCE"
+ }
+}
+```
+
+
+
+If you inspect the full response, you will see that the word `biology` does not appear anywhere.
+
+Even so, Weaviate was able to return biology-related entries. This is made possible by *vector embeddings* that capture meaning. Under the hood, semantic search is powered by vectors, or vector embeddings.
+
+Here is a diagram showing the workflow in Weaviate.
+
+```mermaid
+flowchart LR
+ Query["🔍 Search:\n'biology'"]
+
+ subgraph sg1 ["Vector Search"]
+ direction LR
+ VS1["Convert query\nto vector"] --> VS2["Find similar\nvectors"]
+ VS2 --> VS3["Return top\nmatches"]
+ end
+
+ subgraph sg2 ["Results"]
+ R1["Most similar\ndocuments"]
+ end
+
+ Query --> VS1
+ VS3 --> R1
+
+ %% Style nodes with white background and darker borders
+ style Query fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS3 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style R1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+ %% Style subgraphs with brand colors
+ style sg1 fill:#ffffff,stroke:#61BD73,stroke-width:2px,color:#130C49
+ style sg2 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+```
+
+:::info Where did the vectors come from?
+Weaviate used the locally hosted Ollama model to generate a vector embedding for each object during import. During the query, Weaviate similarly converted the query (`biology`) into a vector.
+
+As we mentioned above, this is optional. See [Starter Guide: Bring Your Own Vectors](/developers/weaviate/starter-guides/custom-vectors.mdx) if you would prefer to provide your own vectors.
+:::
+
+:::tip More search types available
+
+Weaviate is capable of many types of searches. See, for example, our how-to guides on [similarity searches](../search/similarity.md), [keyword searches](../search/bm25.md), [hybrid searches](../search/hybrid.md), and [filtered searches](../search/filters.md).
+
+:::
+
+### 3.2 Retrieval augmented generation
+
+Retrieval augmented generation (RAG), also called generative search, combines the power of generative AI models such as large language models (LLMs) with the up-to-date truthfulness of a database.
+
+RAG works by prompting a large language model (LLM) with a combination of a *user query* and *data retrieved from a database*.
+
+This diagram shows the RAG workflow in Weaviate.
+
+```mermaid
+flowchart LR
+ subgraph sg0 ["Weaviate Query"]
+ direction TB
+ Search["🔍 Search: \n'biology'"]
+ Prompt["✍️ Prompt: \n'Write a\ntweet...'"]
+ end
+
+ subgraph sg1 ["Vector Search"]
+ direction LR
+ VS1["Convert query\nto vector"] --> VS2["Find similar\nvectors"]
+ VS2 --> VS3["Return top\nmatches"]
+ end
+
+ subgraph sg2 ["Generation"]
+ direction LR
+ G1["Send\n(results + prompt)\nto LLM"]
+ G1 --> G2["Generate\nresponse"]
+ end
+
+ subgraph sg3 ["Results"]
+ direction TB
+ R1["Most similar\ndocuments"]
+ R2["Generated\ncontent"]
+ end
+
+ Search --> VS1
+ VS3 --> R1
+ Prompt --> G1
+ VS3 --> G1
+ G2 --> R2
+
+ %% Style nodes with white background and darker borders
+ style Search fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style Prompt fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style VS3 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style G1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style G2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style R1 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+ style R2 fill:#ffffff,stroke:#B9C8DF,color:#130C49
+
+ %% Style subgraphs with brand colors
+ style sg0 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+ style sg1 fill:#ffffff,stroke:#61BD73,stroke-width:2px,color:#130C49
+ style sg2 fill:#ffffff,stroke:#7AD6EB,stroke-width:2px,color:#130C49
+ style sg3 fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49
+```
+
+The following example combines the same search (for `biology`) with a prompt to generate a tweet.
+
+import QueryRAG from '/_includes/code/quickstart/local.quickstart.query.rag.mdx'
+
+
+
+Run this code to perform the query. Here is one possible response (your response will likely be different).
+
+```text
+🧬 In 1953 Watson & Crick built a model of the molecular structure of DNA, the gene-carrying substance! 🧬🔬
+
+🦢 2000 news: the Gunnison sage grouse isn't just another northern sage grouse, but a new species! 🦢🌿 #ScienceFacts #DNA #SpeciesClassification
+```
+
+The response should be new, yet familiar. This because you have seen the entries above for `DNA` and `species` in the [semantic search](#semantic-search) section.
+
+The power of RAG comes from the ability to transform your own data. Weaviate helps you in this journey by making it easy to perform a combined search & generation in just a few lines of code.
+
+
+
+## Recap
+
+In this quickstart guide, you:
+
+- Created a Serverless Weaviate sandbox instance on Weaviate Cloud.
+- Defined a collection and added data.
+- Performed queries, including:
+ - Semantic search, and
+ - Retrieval augmented generation.
+
+Where to go next is up to you. We include some suggested steps and resources below.
+
+
+
+## Next
+
+Try these additional resources to learn more about Weaviate:
+
+
diff --git a/src/components/AboutUsUpdate/styles.module.scss b/src/components/AboutUsUpdate/styles.module.scss
index 7f1b489649..75fa35861d 100644
--- a/src/components/AboutUsUpdate/styles.module.scss
+++ b/src/components/AboutUsUpdate/styles.module.scss
@@ -229,7 +229,7 @@ p {
scroll-snap-type: x mandatory;
white-space: nowrap;
- @media screen and (max-width: 1260px) {
+ @media screen and (max-width: 1440px) {
width: 100%;
}
@@ -253,7 +253,7 @@ p {
width: 65rem;
height: 0.0625rem;
- @media screen and (max-width: 900px) {
+ @media screen and (max-width: 1440px) {
width: 100%;
}
}
diff --git a/src/components/Academy/courseData.js b/src/components/Academy/courseData.js
index 6c3c9f369d..0881fa824c 100644
--- a/src/components/Academy/courseData.js
+++ b/src/components/Academy/courseData.js
@@ -175,6 +175,28 @@ export const courseData = {
],
note: "TS clients"
},
+ "starter_multimodal_typescript": {
+ title: "Multimodal data with Weaviate",
+ courseId: "TS_101M",
+ body: "Project-based learning where you'll learn how to build with Weaviate and multi-modal data. Weaviate generates the vectors for you.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/starter_multimodal_data",
+ badgeType: "course",
+ isCourse: true,
+ units: [
+ "docker_mm_basics_ts", "mm_collections_ts", "mm_searches_ts", "mm_rag_ts"
+ ],
+ learningGoals: [
+ "How to create a Weaviate instance, add data to it to enable multi-modal searching, and use AI through retrieval augmented generation."
+ ],
+ learningOutcomes: [
+ "Create an instance of Weaviate with a multimodal vectorizer module",
+ "Produce, store and index multimodal data",
+ "Perform multimodal searches",
+ "Use AI (large language models) to augment and transform retrieved data",
+ ],
+ note: "TS clients; project-based"
+ },
"multi-tenancy": {
title: "Multi-tenancy",
courseId: "PY_280",
@@ -305,6 +327,20 @@ export const courseData = {
"chunking"
]
},
+ "standalone_js": {
+ title: "Standalone units (JS/TS)",
+ courseId: "0",
+ body: "Bite-sized, standalone units that can be reviewed by themselves.",
+ buttonType: "Notify",
+ badgeType: "course",
+ isCourse: false,
+ units: [
+ "which_search",
+ "client_server",
+ "picking_models",
+ "want_stack"
+ ]
+ },
"zero_to_mvp": {
title: "Zero to MVP: The basics",
courseId: "P3_1",
diff --git a/src/components/Academy/unitData.js b/src/components/Academy/unitData.js
index 300c7d3d7f..831e6f22fa 100644
--- a/src/components/Academy/unitData.js
+++ b/src/components/Academy/unitData.js
@@ -197,6 +197,21 @@ export const unitData = {
"Connect to the Weaviate instance using the Python client."
]
},
+ "docker_mm_basics_ts": {
+ title: "Weaviate for multimodal data",
+ body: "Create a Weaviate instance for multimodal data.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/starter_multimodal_data/setup_weaviate",
+ badgeType: "practical",
+ learningGoals: [
+ "Set up a Weaviate instance with a multimodal module."
+ ],
+ learningOutcomes: [
+ "Create a Weaviate instance with a multimodal module using Docker.",
+ "Install the latest Weaviate TypeScript client.",
+ "Connect to the Weaviate instance using the Typescript client."
+ ]
+ },
"mm_collections": {
title: "Populate the database",
body: "Create a collection and import multimodal data, and have Weaviate create vectors for you.",
@@ -212,6 +227,21 @@ export const unitData = {
"Import data using batch imports."
]
},
+ "mm_collections_ts": {
+ title: "Populate the database",
+ body: "Create a collection and import multimodal data, and have Weaviate create vectors for you.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/starter_multimodal_data/mm_collections",
+ badgeType: "practical",
+ learningGoals: [
+ "Configure and create a collection then import multimodal data using batch imports."
+ ],
+ learningOutcomes: [
+ "Configure a collection with typical settings and vectorizer set.",
+ "Create a collection and work with a collection object.",
+ "Import data using batch imports."
+ ]
+ },
"mm_searches": {
title: "Perform searches",
body: "Learn how to use search functions in Weaviate.",
@@ -226,6 +256,20 @@ export const unitData = {
"Perform searches through multimodal data in different ways.",
]
},
+ "mm_searches_ts": {
+ title: "Perform searches",
+ body: "Learn how to use search functions in Weaviate.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/starter_multimodal_data/mm_searches",
+ badgeType: "practical",
+ learningGoals: [
+ "Perform multimodal searches."
+ ],
+ learningOutcomes: [
+ "Describe how multimodal search works at a high level.",
+ "Perform searches through multimodal data in different ways.",
+ ]
+ },
"mm_rag": {
title: "LLMs and Weaviate (RAG)",
body: "Use large language models to augment and transform retrieved data.",
@@ -241,6 +285,21 @@ export const unitData = {
"Perform RAG on the entire set of returned objects.",
]
},
+ "mm_rag_ts": {
+ title: "LLMs and Weaviate (RAG)",
+ body: "Use large language models to augment and transform retrieved data.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/starter_multimodal_data/mm_rag",
+ badgeType: "practical",
+ learningGoals: [
+ "Use AI (large language models) to augment and transform retrieved data."
+ ],
+ learningOutcomes: [
+ "Describe what RAG is and how it works at a high level.",
+ "Perform RAG on individual objects.",
+ "Perform RAG on the entire set of returned objects.",
+ ]
+ },
"nv_collections": {
title: "Populate the database",
body: "Create a collection with multiple named vectors and import data objects.",
@@ -1006,5 +1065,75 @@ export const unitData = {
],
owner: "jp",
reviewer: "jp"
+ },
+ client_server: {
+ title: "Building client-server Applications",
+ body: "Chunking is essential for working with longer texts in vector databases. This unit covers how to use it as well as tips and best practices.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/standalone/client-server",
+ badgeType: "practical",
+ learningGoals: [
+ "What chunking is",
+ ],
+ learningOutcomes: [
+ "Describe chunking at a high level",
+ "Explain the impact of chunking in vector search and retrieval augmented generation",
+ ],
+ owner: "daniel",
+ reviewer: "daniel"
+ },
+ picking_models: {
+ title: "Using Machine Learning Models",
+ body: "Understand the fundamental concepts of generative and embedding models, their applications, and their role in building AI-Native applications.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/standalone/gen-ai-models",
+ badgeType: "mixed",
+ learningGoals: [
+ "Understand the fundamental concepts of generative and embedding models, their applications, and their role in building AI-Native applications.",
+ ],
+ learningOutcomes: [
+ "The difference between Embedding and Generative models",
+ "When and how to use those models integrated with Weaviate",
+ ],
+ owner: "daniel",
+ reviewer: "daniel"
+ },
+ want_stack: {
+ title: "Understanding the WANT stack?",
+ body: "Chunking is essential for working with longer texts in vector databases. This unit covers how to use it as well as tips and best practices.",
+ buttonType: "Click here",
+ buttonURL: "/developers/academy/js/standalone/want-stack",
+ badgeType: "practical",
+ learningGoals: [
+ "What chunking is",
+ ],
+ learningOutcomes: [
+ "Describe chunking at a high level",
+ "Explain the impact of chunking in vector search and retrieval augmented generation",
+ ],
+ owner: "daniel",
+ reviewer: "daniel"
+ },
+ which_search_js: {
+ title: "Which search is right for me?",
+ body: "Weaviate offers many search types (vector, keyword and hybrid), and options. Let's discuss some good, baseline search strategies.",
+ buttonType: "Click here",
+ badgeType: "mixed",
+ buttonURL: "/developers/academy/js/standalone/which-search",
+ learningGoals: [
+ "Impact of search type on search quality.",
+ "Impact of search type on search performance.",
+ "How the dataset and chunking affect search",
+ "Key considerations for selecting a search type.",
+ "Strategies to apply to improve search quality.",
+ ],
+ learningOutcomes: [
+ "Broadly recite pros and cons of each search type (vector, keyword and hybrid).",
+ "Suggest a suitable search type given a description of the dataset and aim.",
+ "Suggest alternative or additional search strategies to improve search quality.",
+ "Outline broad methods to evaluate search quality."
+ ],
+ owner: "daniel",
+ reviewer: "daniel"
}
};
diff --git a/src/components/Blog/styles.module.scss b/src/components/Blog/styles.module.scss
index 3550fc6aab..8a146dafd1 100644
--- a/src/components/Blog/styles.module.scss
+++ b/src/components/Blog/styles.module.scss
@@ -12,3 +12,15 @@
top: 100px;
}
}
+
+[data-theme='light'] {
+ .blogHome {
+ color: #525860;
+ }
+}
+
+[data-theme='dark'] {
+ .blogHome {
+ color: #ffffff;
+ }
+}
diff --git a/src/components/CaseStudies/Header/index.jsx b/src/components/CaseStudies/Header/index.jsx
new file mode 100644
index 0000000000..2f12e7758b
--- /dev/null
+++ b/src/components/CaseStudies/Header/index.jsx
@@ -0,0 +1,73 @@
+import Link from '@docusaurus/Link';
+import React from 'react';
+import styles from './styles.module.scss';
+import { LinkButton, ButtonContainer } from '/src/theme/Buttons';
+
+export default function StudyHeader() {
+ return (
+
+
+
+
+ Case Study
+
+ How Morningstar built a trustworthy, AI-driven financial data
+ platform
+
+ Instabase is an enterprise-grade AI Application Platform that
+ turns unstructured data into insights, instantly. They solve one
+ of the most acute problems businesses face – slow or inaccurate
+ decisioning due to unstructured data. Their engineering team was
+ looking for a database that would allow them to deliver data
+ insights to their users with a high degree of accuracy and rapid
+ performance. They chose Weaviate because of the flexibility that a
+ leading open-source tool gave them while hitting Instabase’s
+ critical performance metrics better than any other database they
+ tested. Built-in features like hybrid search and distance metrics,
+ as well as support from an engaged developer community, made
+ Weaviate the clear choice for Instabase.
+
+
Challenge
+
+ Instabase processes over 500K highly varied documents per day and
+ needs to scale to the ever-growing demand of their customers. They
+ needed a database solution that would allow them to index, store,
+ and retrieve massive volumes of data while delivering results with
+ an incredibly high level of accuracy to their users. “Accuracy
+ determines the amount of savings any large institution can get,”
+ said Shaunak Godbole, Head of Infrastructure Engineering at
+ Instabase. “If the results aren’t accurate or take too long to
+ surface, a human needs to get involved, and the cost savings goes
+ away. So accuracy and speed are critical for us.” Additionally,
+ Instabase needed to support customers in highly regulated
+ environments. European customers needed to ensure their data
+ didn’t leave certain countries, while financial institutions
+ didn’t want their data to leave their on-premises servers. They
+ needed a database solution that could be deployed anywhere while
+ maintaining a high level of performance. If accuracy, speed, and
+ flexible deployment criteria were not met, Instabase would not be
+ able to reduce the need for human intervention in complex data
+ workflows for their customers.
+
+
+
Why Instabase Chose Weaviate
+
+ Instabase wants to empower their customers to focus on making fast
+ and accurate decisions. They achieve this by classifying,
+ extracting, and validating information from highly unstructured
+ data. “The collaboration with Weaviate’s team, the community, and
+ the results of our performance tests made Weaviate an easy
+ choice,” notes Godbole. “In terms of performance, nobody needed
+ convincing – the benchmarks spoke for themselves.”
+
+
Weaviate checked all of Instabase’s boxes:
+
+
+ Powerful performance: Instabase had their own
+ benchmarks for the use cases they knew they had to solve really,
+ really well. These benchmarks focus on retrieval accuracy and
+ latency, and the queries are complex aggregation and composition
+ where both dense and sparse searches are required. Each use case
+ had different benchmarks, and Weaviate was a clear winner with
+ high retrieval accuracy and low latency for vector search. Once
+ the benchmarking team was convinced Weaviate was up for the
+ challenge, teams across the organization worked together to
+ bring Weaviate into production.
+
+
+ High adaptability: By using an AI-native
+ open-source vector database, Instabase could meet their
+ customers wherever they operated, whether in the cloud or
+ on-prem. This allowed for maximum flexibility to fulfill the
+ strict deployment needs of organizations in highly regulated
+ regions and industries.
+
+
+ Modular architecture: Instead of having to
+ build out their own capabilities, Instabase developers saved
+ time by using Weaviate’s out of the box features like hybrid
+ search and distance metrics, as well as easy integrations with
+ popular large language models (LLMs).
+
+
+ Strong support: Weaviate has cultivated an
+ engaged, open source community with over 6M downloads and tens
+ of thousands of organizations using the platform. Over 5K
+ developers have direct access to the experts and comprehensive
+ documentation they need to quickly problem solve as they build
+ their applications. In addition, Weaviate’s core team was very
+ collaborative with Instabase and was able to provide short,
+ medium and long-term solutions to existing and anticipated
+ challenges.
+
+
+
+
+
+ “Accuracy determines the amount of savings any large institution
+ can get. If the results aren’t accurate or take too long to
+ surface, a human needs to get involved, and the cost savings are
+ greatly reduced. So accuracy and speed are critical for us.”
+
+
+
+ Shaunak Godbole, Head of Infrastructure Engineering at Instabase
+
+
+
+
+
+
Rapid performance
+
+ Instabase is able to store 50K+ tenants in the Weaviate cluster
+ and query data from specific tenants within milliseconds.
+
+
Scales to unlimited document size
+
+ Instabase has seen the same impressive results whether customers
+ engage with single-page handwritten notes, 200 pages of
+ documentation, or 400 pages of financial filings.
+
+
450+ data types supported
+
+ With Weaviate, Instabase was able to support the ingestion and
+ indexing of 450+ data types for a single customer solution.
+
+
+
+
+
+
+ Instabase helps
+ organizations across large enterprises, mid-market companies, and
+ the federal government solve mission-critical automation problems
+ and obtain insights from unstructured data. Their platform, called
+ AI Hub, enables customers to obtain instant insights in Converse
+ mode. When in Build mode, they can extract data from any content,
+ classify, clean it up and embed the newly structured data
+ downstream into existing systems. These automations enable
+ customers to quickly and accurately make vital business decisions
+ for complex processes like mortgage underwriting, driver’s license
+ verification, or insurance broker submissions. Instabase’s
+ customers include the world’s largest financial institutions,
+ insurance companies, transportation, retail, and public sector
+ organizations.
+
+
Why AWS
+
+ In 2017, the engineers at Instabase made the choice to build
+ entirely on Kubernetes. They have several microservices that run
+ on top of that and are deeply integrated with the{' '}
+
+ AWS ecosystem
+
+ , making use of products including AWS EC2, ECS, EKS, ELB, S3,
+ ElastiCache, CDN, and Shield. AWS allows Instabase to build a
+ highly-available, secure, scalable, and performant platform to
+ support even their largest customers.
+
+
About Weaviate
+
+ Vector databases are becoming core to the AI tech stack because
+ they can handle a very large amount of unstructured data in an
+ efficient way. Weaviate is an AI-native vector database available
+ on the AWS marketplace that can scale to handle billions of
+ vectors and millions of tenants. Customers and community members
+ use Weaviate to power large-scale search and generative AI
+ applications like chatbots and agents. Weaviate’s extensible
+ architecture offers easy pluggability with the AI ecosystem,
+ empowering developers of all levels to build and iterate faster.
+ And flexible deployment options let teams abstract the burden of
+ hosting and managing their database, while still meeting
+ enterprise requirements for security and compliance.
+
+ {' '}
+ Morningstar selected Weaviate's vector database to build a
+ scalable, accurate Intelligence Engine Platform, delivering
+ reliable AI-driven financial tools and dynamic document search.
+
+
+
+
+
+
Driving Innovation
+
+
+ The Intelligence Engine powers hundreds of in-house applications.
+
+
+
+
RAG pipelines
+
+
+ The Intelligence Engine powers hundreds of in-house applications.
+
+
+
+
+
+
Financial Research
+
+
+ The Intelligence Engine powers hundreds of in-house applications.
+
+
+
+
Self-serve RAG
+
+
+ The Intelligence Engine powers hundreds of in-house applications.
+
+
+
+
+
+
+
+ about the company
+
+
+ Morningstar, Inc.
+ is a leading global provider of independent investment insights.
+ Morningstar offers an extensive line of products and services
+ for individual investors, financial advisors, asset managers and
+ owners, retirement plan providers and sponsors, and
+ institutional investors in the debt and private capital markets.
+ Morningstar provides data and research insights on a wide range
+ of investment offerings, including managed investment products,
+ publicly listed companies, private capital markets, debt
+ securities, and real-time global market data.
+
+
+
+
+
+ challenge
+
+ Over the last 40 years, Morningstar amassed an extensive
+ collection of proprietary financial data. Morningstar has sought
+ to further empower investors that rely on its data by developing
+ an advanced research assistant AI application. According to
+ Benjamin Barrett, Morningstar’s Head of Technology, Research
+ Products, building a chatbot on that data “looks like magic, but
+ when you start peeling back the layers of the onion, you have to
+ ask, is it actually accurate? Is it pulling the latest,
+ greatest, most relevant data? Are our answers robust and
+ complete?”
+
+
+ As his engineering team worked with Morningstar’s Quantitative
+ Research team to build their Intelligence Engine Platform, they
+ had to ensure an incredibly high level of accuracy in order to
+ maintain their users’ trust. “We want to have one single source
+ of truth. Our whole mission is to empower investor success. And
+ the way to do that is to give them reliable, trustworthy
+ financial data.”
+
+
+
+
+
+
+ why weaviate
+
+ In early 2023, Morningstar saw early success in experiments with
+ LLMs and snippets of their own data. They quickly realized the
+ potential of using AI to harness decades of longform research
+ content and real-time data with RAG, and started their search
+ for the right vector database.
+
+
+
+
+
+ Ease-of-use: Weaviate’s open-source database
+ was quick and easy for Morningstar to spin up locally in a
+ Docker container and start experimenting.
+
+
+
+
+
+ Data privacy and security: Flexible deployment
+ options and multi-tenant architecture allowed for strict data
+ privacy and security compliance.
+
+
+
+
+
+ Flexibility and scalability: Weaviate supported
+ a variety of use cases–from search engines to tailored AI
+ applications–and was able to handle large and diverse data sets.
+
+
+
+
+
+ Support: Weaviate offered great support from
+ local development all the way to production.
+
+
+
+
+
+
+ results
+
+ With Weaviate, Morningstar was able to build their
+ Intelligence Engine Platform, a product which solves a
+ challenge facing today’s financial services firms: how to
+ easily create and customize AI applications built on a
+ foundation of trusted financial data and research. The
+ Intelligence Engine also powers a variety of workflows, APIs,
+ and chat interfaces across Morningstar’s own product ecosystem
+ – including Mo, it’s investment research-assistant chatbot –
+ for both internal and external users.
+
+
+
+
+
+
+
We’re here to help!
+
+ Need support getting started with Weaviate? We’ve got your
+ back.
+
+
+
+
+
+ {'Get in touch >'}
+
+
+
+
+
+
+
+
+ We’d love to give you a live demo of what Weaviate can do
+
+ Instabase is an enterprise-grade AI Application Platform that
+ turns unstructured data into insights, instantly. They solve one
+ of the most acute problems businesses face – slow or inaccurate
+ decisioning due to unstructured data. Their engineering team was
+ looking for a database that would allow them to deliver data
+ insights to their users with a high degree of accuracy and rapid
+ performance. They chose Weaviate because of the flexibility that a
+ leading open-source tool gave them while hitting Instabase’s
+ critical performance metrics better than any other database they
+ tested. Built-in features like hybrid search and distance metrics,
+ as well as support from an engaged developer community, made
+ Weaviate the clear choice for Instabase.
+
+
Challenge
+
+ Instabase processes over 500K highly varied documents per day and
+ needs to scale to the ever-growing demand of their customers. They
+ needed a database solution that would allow them to index, store,
+ and retrieve massive volumes of data while delivering results with
+ an incredibly high level of accuracy to their users. “Accuracy
+ determines the amount of savings any large institution can get,”
+ said Shaunak Godbole, Head of Infrastructure Engineering at
+ Instabase. “If the results aren’t accurate or take too long to
+ surface, a human needs to get involved, and the cost savings goes
+ away. So accuracy and speed are critical for us.” Additionally,
+ Instabase needed to support customers in highly regulated
+ environments. European customers needed to ensure their data
+ didn’t leave certain countries, while financial institutions
+ didn’t want their data to leave their on-premises servers. They
+ needed a database solution that could be deployed anywhere while
+ maintaining a high level of performance. If accuracy, speed, and
+ flexible deployment criteria were not met, Instabase would not be
+ able to reduce the need for human intervention in complex data
+ workflows for their customers.
+
+
+
Why Instabase Chose Weaviate
+
+ Instabase wants to empower their customers to focus on making fast
+ and accurate decisions. They achieve this by classifying,
+ extracting, and validating information from highly unstructured
+ data. “The collaboration with Weaviate’s team, the community, and
+ the results of our performance tests made Weaviate an easy
+ choice,” notes Godbole. “In terms of performance, nobody needed
+ convincing – the benchmarks spoke for themselves.”
+
+
Weaviate checked all of Instabase’s boxes:
+
+
+ Powerful performance: Instabase had their own
+ benchmarks for the use cases they knew they had to solve really,
+ really well. These benchmarks focus on retrieval accuracy and
+ latency, and the queries are complex aggregation and composition
+ where both dense and sparse searches are required. Each use case
+ had different benchmarks, and Weaviate was a clear winner with
+ high retrieval accuracy and low latency for vector search. Once
+ the benchmarking team was convinced Weaviate was up for the
+ challenge, teams across the organization worked together to
+ bring Weaviate into production.
+
+
+ High adaptability: By using an AI-native
+ open-source vector database, Instabase could meet their
+ customers wherever they operated, whether in the cloud or
+ on-prem. This allowed for maximum flexibility to fulfill the
+ strict deployment needs of organizations in highly regulated
+ regions and industries.
+
+
+ Modular architecture: Instead of having to
+ build out their own capabilities, Instabase developers saved
+ time by using Weaviate’s out of the box features like hybrid
+ search and distance metrics, as well as easy integrations with
+ popular large language models (LLMs).
+
+
+ Strong support: Weaviate has cultivated an
+ engaged, open source community with over 6M downloads and tens
+ of thousands of organizations using the platform. Over 5K
+ developers have direct access to the experts and comprehensive
+ documentation they need to quickly problem solve as they build
+ their applications. In addition, Weaviate’s core team was very
+ collaborative with Instabase and was able to provide short,
+ medium and long-term solutions to existing and anticipated
+ challenges.
+
+
+
+
+
+ “Accuracy determines the amount of savings any large institution
+ can get. If the results aren’t accurate or take too long to
+ surface, a human needs to get involved, and the cost savings are
+ greatly reduced. So accuracy and speed are critical for us.”
+
+
+
+ Shaunak Godbole, Head of Infrastructure Engineering at Instabase
+
+
+
+
+
+
Rapid performance
+
+ Instabase is able to store 50K+ tenants in the Weaviate cluster
+ and query data from specific tenants within milliseconds.
+
+
Scales to unlimited document size
+
+ Instabase has seen the same impressive results whether customers
+ engage with single-page handwritten notes, 200 pages of
+ documentation, or 400 pages of financial filings.
+
+
450+ data types supported
+
+ With Weaviate, Instabase was able to support the ingestion and
+ indexing of 450+ data types for a single customer solution.
+
+
+
+
+
+
+ Instabase helps
+ organizations across large enterprises, mid-market companies, and
+ the federal government solve mission-critical automation problems
+ and obtain insights from unstructured data. Their platform, called
+ AI Hub, enables customers to obtain instant insights in Converse
+ mode. When in Build mode, they can extract data from any content,
+ classify, clean it up and embed the newly structured data
+ downstream into existing systems. These automations enable
+ customers to quickly and accurately make vital business decisions
+ for complex processes like mortgage underwriting, driver’s license
+ verification, or insurance broker submissions. Instabase’s
+ customers include the world’s largest financial institutions,
+ insurance companies, transportation, retail, and public sector
+ organizations.
+
+
Why AWS
+
+ In 2017, the engineers at Instabase made the choice to build
+ entirely on Kubernetes. They have several microservices that run
+ on top of that and are deeply integrated with the{' '}
+
+ AWS ecosystem
+
+ , making use of products including AWS EC2, ECS, EKS, ELB, S3,
+ ElastiCache, CDN, and Shield. AWS allows Instabase to build a
+ highly-available, secure, scalable, and performant platform to
+ support even their largest customers.
+
+
About Weaviate
+
+ Vector databases are becoming core to the AI tech stack because
+ they can handle a very large amount of unstructured data in an
+ efficient way. Weaviate is an AI-native vector database available
+ on the AWS marketplace that can scale to handle billions of
+ vectors and millions of tenants. Customers and community members
+ use Weaviate to power large-scale search and generative AI
+ applications like chatbots and agents. Weaviate’s extensible
+ architecture offers easy pluggability with the AI ecosystem,
+ empowering developers of all levels to build and iterate faster.
+ And flexible deployment options let teams abstract the burden of
+ hosting and managing their database, while still meeting
+ enterprise requirements for security and compliance.
+
Our integrations to{' '}
build generative ai tools
-
- {' '}work just as well with your data and vectors.
+ {' '}
+ work just as well with your data and vectors.
+ With Weaviate, you can bring your own vectors or choose one of our
+ out-of-the-box modules with support for vectorization. You can
+ also easily connect to a wide variety of well-known language model
+ frameworks.
+
+ Intermediate level workshops, or webinars with industry users
+
+
+
+
+ Intro to Hybrid Search:
+ {' '}
+ Combining the power of keyword and vector search
+
+
+
+ Build AI apps with Weaviate's TypeScript client:
+ {' '}
+ Vue.js, React.js and Angular.js
+
+
+
+
+ More workshops & webinars
+
+
+
+
+
+
+
+ Live online workshop
+
Intro to building applications with Weaviate
+
+
+ Learn why vector databases are great, how to create one with
+ Weaviate, and how to use it with real data, including a
+ demo app and an end-to-end Jupyter notebook example.
+
All the power of the AI-native vector database, without the
overhead
diff --git a/src/components/Pricing/Header/styles.module.scss b/src/components/Pricing/Header/styles.module.scss
index 7248b34000..1df70074fd 100644
--- a/src/components/Pricing/Header/styles.module.scss
+++ b/src/components/Pricing/Header/styles.module.scss
@@ -89,11 +89,10 @@
flex-direction: row;
flex-wrap: wrap;
align-items: center;
- justify-content: center;
+
+ justify-content: flex-start;
a {
- height: 45px;
- width: 220px;
display: flex;
align-items: center;
justify-content: center;
@@ -113,6 +112,13 @@
}
.buttonOutline {
+ width: 20.0625rem !important;
+ color: #130c49;
+ text-align: center;
+
+ font-weight: 600;
+
+ height: 1.375rem;
border-radius: 0.3125rem;
color: #130c49;
background: #fff;
diff --git a/src/components/Pricing/Plan/businessCritical.jsx b/src/components/Pricing/Plan/businessCritical.jsx
index b21375b04c..545a94955a 100644
--- a/src/components/Pricing/Plan/businessCritical.jsx
+++ b/src/components/Pricing/Plan/businessCritical.jsx
@@ -38,7 +38,8 @@ export default function PricingBusinessCritical() {
- Choose a fully-managed solution or 24/7 support within your VPC.
+ Choose a fully-managed solution or 24/7 support within your VPC
+ (BYOC Vector Database).
@@ -51,19 +52,18 @@ export default function PricingBusinessCritical() {
-
- For building and prototyping with seamless scaling and flexible
- pay-as-you-go pricing.
-
+
For running workflows within your Virtual Private Cloud (VPC).
- Get started with a free trial in minutes
+ Weaviate-managed control plane
- Various SLA tiers to meet your needs
+
+ Weaviate agent for monitoring, support, and troubleshooting
+
diff --git a/src/components/Pricing/Plan/enterprise.jsx b/src/components/Pricing/Plan/enterprise.jsx
index e6de740180..0f888a7a58 100644
--- a/src/components/Pricing/Plan/enterprise.jsx
+++ b/src/components/Pricing/Plan/enterprise.jsx
@@ -1,10 +1,11 @@
-import React, { useState, useEffect } from 'react';
+import React, { useState, useEffect, useRef } from 'react';
import styles from './styles.module.scss';
import Link from '@docusaurus/Link';
import EnterpriseContainer from '../EnterpriseContainer';
export default function PricingEnterprise() {
const [isModalOpen, setIsModalOpen] = useState(false);
+ const modalRef = useRef(null); // Ref for the modal content
const openModal = () => {
if (window.location.pathname === '/pricing') {
@@ -20,6 +21,23 @@ export default function PricingEnterprise() {
}
};
+ const handleOutsideClick = (e) => {
+ if (modalRef.current && !modalRef.current.contains(e.target)) {
+ closeModal();
+ }
+ };
+
+ useEffect(() => {
+ if (isModalOpen) {
+ document.addEventListener('mousedown', handleOutsideClick);
+ } else {
+ document.removeEventListener('mousedown', handleOutsideClick);
+ }
+ return () => {
+ document.removeEventListener('mousedown', handleOutsideClick);
+ };
+ }, [isModalOpen]);
+
useEffect(() => {
if (window.location.pathname === '/pricing/enterprise') {
setIsModalOpen(true);
@@ -49,7 +67,7 @@ export default function PricingEnterprise() {
We manage everything for you in a dedicated instance in Weaviate
- Cloud.
+ Cloud (Enterprise Vector Database).
from $2.64 / AIU
@@ -79,12 +97,12 @@ export default function PricingEnterprise() {
-
+
×
-
diff --git a/src/components/Pricing/Plan/sandbox.jsx b/src/components/Pricing/Plan/sandbox.jsx
index 676b7ad9d3..4b1583cb36 100644
--- a/src/components/Pricing/Plan/sandbox.jsx
+++ b/src/components/Pricing/Plan/sandbox.jsx
@@ -12,7 +12,8 @@ export default function PricingSandBox() {
With an enterprise subscription, customers have access to a variety of
- support and training options to accelerate adoption and success.
+ support and vector database training options to accelerate adoption
+ and success.
- Weaviate Cloud was built to offload the burden of managing
- a vector database and give development teams peace of mind. Whether
- you’re running Weaviate as a Serverless, Weaviate Enterprise Cloud, or
+ Weaviate Cloud was built to offload the burden of managing a vector
+ database and give development teams peace of mind. Whether you’re
+ running Weaviate as a Serverless, Weaviate Enterprise Cloud, or
Bring Your Own Cloud solution– our top priority is keeping your data
safe and available. Read more about our SLAs{' '}
- here.
+ here. You can also read our Security
+ Checklist for Self-Managed Weaviate Deployments{' '}
+
+ here
+
+ .
diff --git a/src/components/Service/BYOC/Integrations/index.jsx b/src/components/Service/BYOC/Integrations/index.jsx
index e1305bdb8f..56fa0b0ea2 100644
--- a/src/components/Service/BYOC/Integrations/index.jsx
+++ b/src/components/Service/BYOC/Integrations/index.jsx
@@ -18,7 +18,7 @@ export default function Integrations() {
Run Weaviate within your VPC
Take advantage of all the best{' '}
- vector search{' '}
+ vector search{' '}
and RAG techniques while ensuring your data stays safe within your
own cloud environment.
diff --git a/src/components/Service/Finance/Header/index.jsx b/src/components/Service/Finance/Header/index.jsx
new file mode 100644
index 0000000000..f8f2c97b5f
--- /dev/null
+++ b/src/components/Service/Finance/Header/index.jsx
@@ -0,0 +1,20 @@
+import Link from '@docusaurus/Link';
+import React from 'react';
+import styles from './styles.module.scss';
+import { LinkButton, ButtonContainer } from '/src/theme/Buttons';
+
+export default function StudyHeader() {
+ return (
+
+
+
+ Case Study
+
+ How a Leading Financial Data Company Commercialized AI in Under a
+ Year
+
+
+ A leading US financial data analytics company needed to enable
+ their engineering teams to build and scale secure, reliable AI
+ applications, fast. Weaviate’s AI-native, open source vector
+ database came with robust out-of-the-box features, flexible
+ deployment options, strong performance, and reliable support,
+ making it the company’s vector database of choice. In less than
+ one year, this company commercialized AI and empowered every
+ internal employee to ask questions about their files in their
+ proprietary chat tool.
+
+
+
Challenge
+
+ A Lead System Engineer (LSE) was tasked to determine which vector
+ database would best enable engineering teams across the company to
+ build and scale secure applications in production. As AI
+ excitement spread across the financial data industry, he noticed
+ that within a month there were four to five different vector
+ databases in use across internal teams. He needed to find and
+ implement the right solution before tool sprawl got out of hand
+ and his team would be stuck supporting a bloated tech stack.
+ Additionally, competitors were starting to build their own AI
+ tools to support faster and more accurate decision making – the
+ company needed to move quickly to maintain the best-in-class
+ service they provide their customers.
+
+
Why Weaviate
+
+ Early in his research, the LSE learned that his company’s
+ engineers weren’t just looking for a database to support basic
+ search features – they wanted to build chatbots, integrate
+ multiple data sources and large language models (LLMs), have
+ support for vector search, metadata, hybrid search, reranking, and
+ filtering. He worked across teams to determine the importance of
+ ease of implementation, performance benchmarks and infrastructure
+ requirements like backups, restoration, and being able to deploy a
+ vector database on premises or in their own AWS cloud environment.
+ Many vector databases could meet multiple requirements, but only
+ Weaviate could meet them all.{' '}
+
+
+ “Experimenting is one thing, but when you’re building long-term,
+ client-facing enterprise applications, you want the right vector
+ database with the right level of support. Weaviate was that
+ database for us,”
+
+
+ Lead System Engineer from a top US financial data analytics
+ company
+
+
+
Solution
+
+ The company chose Weaviate because it fulfilled four primary
+ requirements:
+
+
+
+ Robust feature set: Weaviate’s modular
+ architecture allowed easy integration of various data types and
+ sources and popular LLMs. Out-of-the-box features like hybrid
+ search, reranking, and filtering allowed developers of various
+ skill levels across teams to focus on building applications
+ instead of creating those functions from scratch.
+
+
+ Flexible deployment options: The company could
+ deploy Weaviate on AWS EKS through a pre-vetted Kubernetes
+ blueprint for internal usage, fulfilling security requirements
+ and allowing teams to move quickly. As an open-source vector
+ database, Weaviate could be deployed on premises and in the
+ cloud allowing strict data control.
+
+
+ Meeting required benchmarks: The infrastructure
+ team surveyed internal teams about the technical requirements
+ for various use cases – including queries per second, speed of
+ implementation, and high availability. Weaviate was able to
+ perform well within their required benchmark ranges.
+
+
+ Reliable support: The company has an open
+ culture of sharing what’s working and not working in their Slack
+ channels for third-party developer tools. The Weaviate channel
+ became a source of internal collaboration, problem solving, and
+ best practice sharing with the Weaviate team. In addition, the
+ company has peace-of-mind knowing they can access Weaviate’s
+ 24/7 support team.
+
+
+
+
+
+
+
{'<1 year to commercialized AI'}
+
+ “Our whole value as a company is our expertise and content. We
+ were able to translate the speed of implementation of Weaviate to
+ the speed of delivering commercial AI products within a year,”
+ said the LSE.
+
+
Unlocking AI for non-technical roles
+
+ “Every employee can now upload files and ask questions about them
+ in our internal chat tool. Anyone, even if they're not an engineer
+ and know nothing about coding, is able to use really good RAG.”
+ said a VP, Principal Software Engineer who developed a data
+ platform built on Weaviate.
+
+
+
+
+
+
+ “Teams started flocking to it. I thought I would need to convince
+ them to move from Pinecone, ChromaDB or Postgres. But as internal
+ expertise grew, people just started adopting Weaviate for its
+ features, performance, and ease of use.”
+
+
+ Lead System Engineer from a top US financial data analytics
+ company
+
- In 2017, the engineers at Instabase made the choice to build
- entirely on Kubernetes. They have several microservices that run
- on top of that and are deeply integrated with the{' '}
-
- AWS ecosystem
-
- , making use of products including AWS EC2, ECS, EKS, ELB, S3,
- ElastiCache, CDN, and Shield. AWS allows Instabase to build a
- highly-available, secure, scalable, and performant platform to
- support even their largest customers.
-
+
About Weaviate
Vector databases are becoming core to the AI tech stack because
they can handle a very large amount of unstructured data in an
- efficient way. Weaviate is an AI-native vector database available
- on the AWS marketplace that can scale to handle billions of
- vectors and millions of tenants. Customers and community members
- use Weaviate to power large-scale search and generative AI
- applications like chatbots and agents. Weaviate’s extensible
- architecture offers easy pluggability with the AI ecosystem,
- empowering developers of all levels to build and iterate faster.
- And flexible deployment options let teams abstract the burden of
- hosting and managing their database, while still meeting
- enterprise requirements for security and compliance.
+ efficient way. Weaviate is an AI-native vector database that can
+ scale to handle billions of vectors and millions of tenants.
+ Customers and community members use Weaviate to power large-scale
+ search and generative AI applications like chatbots and agents.
+ Weaviate’s extensible architecture offers easy pluggability with
+ the AI ecosystem, empowering developers of all levels to build and
+ iterate faster. And flexible deployment options let teams abstract
+ the burden of hosting and managing their database, while still
+ meeting enterprise requirements for security and compliance.
-
+
@@ -102,7 +100,7 @@ export default function introduction() {
Have questions about pricing?
- Contact us
+ Contact us
diff --git a/src/components/Signup/Demo/introduction.jsx b/src/components/Signup/Demo/introduction.jsx
new file mode 100644
index 0000000000..66c2cea406
--- /dev/null
+++ b/src/components/Signup/Demo/introduction.jsx
@@ -0,0 +1,119 @@
+import Link from '@docusaurus/Link';
+import React, { useEffect } from 'react';
+import styles from './styles.module.scss';
+import { css } from 'styled-components';
+
+export default function Introduction() {
+ useEffect(() => {
+ // Load the external HubSpot form script
+ const script = document.createElement('script');
+ script.src = '//js.hsforms.net/forms/embed/v2.js';
+ script.async = true;
+ script.onload = () => {
+ if (window.hbspt) {
+ window.hbspt.forms.create({
+ portalId: '8738733',
+ formId: '62449f1d-d31f-44b3-b52d-9903f1ed448e',
+ target: '#hs-form',
+ });
+ }
+ };
+ document.body.appendChild(script);
+ }, []);
+
+ return (
+
+
+
+ Learn how to take your AI applications
+ to market faster with Weaviate
+
+
+
+ See how Weaviate’s AI-native vector database gives AI builders the
+ tools needed to move fast—without compromising data privacy,
+ performance, or costs.
+
+
+
+ Sign up for a 1:1 call with our team to see how to:
+
+
+
+
+
+
+
+ Blend vector and keyword search (with no extra work) for
+ accurate, contextual results across every type of data.
+
+
+
+
+
+
+
+ Connect to your LLM of choice and easily switch models for fast,
+ flexible RAG development.
+
+
+
+
+
+
+
+ Reduce resource consumption with techniques like vector
+ compression and data offloading.
+
+
+
+
+
+
+
+ “Through our corpus API connected to Weaviate, users can build
+ very powerful, low latency search engines in minutes with little
+ to no code.”
+
+ Learn how to take your AI applications
+ to market faster with Weaviate
+
+
+
+ See how Weaviate’s AI-native vector database gives AI builders the
+ tools needed to move fast—without compromising data privacy,
+ performance, or costs.
+
+
+
+ Sign up for a 1:1 call with our team to see how to:
+
+
+
+
+
+
+
+ Blend vector and keyword search (with no extra work) for
+ accurate, contextual results across every type of data.
+
+
+
+
+
+
+
+ Connect to your LLM of choice and easily switch models for fast,
+ flexible RAG development.
+
+
+
+
+
+
+
+ Reduce resource consumption with techniques like vector
+ compression and data offloading.
+
+
+
+
+
+
+
+ “Through our corpus API connected to Weaviate, users can build
+ very powerful, low latency search engines in minutes with little
+ to no code.”
+
+ Stay ahead on the future of AI-native software development
+
+
+ Vector databases are quickly emerging as a key component of the
+ AI-native tech stack, enabling fundamental use cases like
+ semantic search, recommendation engines, and retrieval augmented
+ generation (RAG).{' '}
+
+
+
+ This Refcard provides an overview of vector databases, including
+ basic concepts, use cases, and examples.
+
+
+ Weaviate named a Leader and Fast-Mover{' '}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ “[Weaviate] is in the Leader circle because of cost-saving
+ techniques that include storage tiering, its use of language models
+ for summarization and NLQ, and a modular approach to accessing
+ models for embedding content and ranking search results.”
+
+
+ Gigaom Sonar Report for Vector Databases by Andrew Brust and Jelani
+ Harper, December 11. 2024
+
+
+
+
About the report
+
+ The GigaOm Sonar Report for Vector Databases provides a thorough
+ examination of the rapidly evolving vector database market. In its
+ second edition, the report equips IT decision-makers with detailed
+ insights into vendor offerings and evaluation criteria, including:
+
+ This page lists the subprocessors that we use to provide our
+ services. Subprocessors are third-party data processors that we use
+ to provide our services.
+
+
+
+
+ );
+}
diff --git a/src/components/Subprocessors/Header/styles.module.scss b/src/components/Subprocessors/Header/styles.module.scss
new file mode 100644
index 0000000000..f3af6d6185
--- /dev/null
+++ b/src/components/Subprocessors/Header/styles.module.scss
@@ -0,0 +1,30 @@
+.box {
+ display: flex;
+ flex-direction: column;
+ justify-content: center;
+ align-items: center;
+ text-align: center;
+ margin: 0px 0px 40px 0px;
+
+ h1 {
+ color: #130c49;
+ text-align: center;
+ font-family: Plus Jakarta Sans;
+ font-size: 2.25rem;
+ font-style: normal;
+ font-weight: 700;
+ line-height: 140%; /* 3.15rem */
+ margin-bottom: 1rem;
+ }
+
+ .headerBox {
+ width: 50%;
+ @media screen and (max-width: 900px) {
+ width: 100%;
+ }
+
+ p {
+ color: #130c49;
+ }
+ }
+}
diff --git a/src/css/custom.scss b/src/css/custom.scss
index 1fd73e60ef..cadea1d2b8 100644
--- a/src/css/custom.scss
+++ b/src/css/custom.scss
@@ -529,7 +529,7 @@ button.close {
img {
display: none;
}
- background-image: url('/img/site/weaviate-nav-logo-light.svg');
+ background-image: url('/img/site/weaviate-logo-horizontal-dark-1.svg');
}
}
@@ -560,20 +560,24 @@ button.close {
}
.navbar {
- background: linear-gradient(
- 90deg,
- rgba(0, 0, 0, 1) 4%,
- rgba(17, 11, 62, 1) 50%,
- rgba(25, 16, 92, 1) 96%
- );
+ background: var(--site-background);
box-shadow: none;
+ border-bottom: solid var(--ifm-toc-border-color);
+ border-width: 1px;
}
.navbar__logo {
img {
display: none;
}
- background-image: url('/img/site/weaviate-nav-logo-dark.svg');
+
+ background-image: url('/img/site/weaviate-logo-horizontal-light-1.svg');
+ background-size: contain;
+ background-repeat: no-repeat;
+ }
+
+ .blogContainer {
+ background-color: #0c1428;
}
.footer--dark {
@@ -752,13 +756,13 @@ Dropdown style removed.
.custom-page .footer__links:before {
content: '';
- background-image: url('/img/site/weaviate-nav-logo-dark.svg');
+ background-image: url('/img/site/weaviate-logo-horizontal-light-1.svg');
background-size: contain;
background-repeat: no-repeat;
- width: 11rem;
- height: 11rem;
- margin: -21px 0px 0px 0px;
+ height: 2rem;
+ margin: -1px 0 0;
+ width: 8rem;
}
.content_node_modules-\@docusaurus-theme-classic-lib-theme-AnnouncementBar-Content-styles-module {
@@ -1026,3 +1030,7 @@ ul.dropdown__menu {
.loadingIcon {
margin: auto;
}
+
+.support-page li {
+ font-size: 1.2rem;
+}
diff --git a/src/css/shared.scss b/src/css/shared.scss
index 3b9bfe4962..4f43c7eaf8 100644
--- a/src/css/shared.scss
+++ b/src/css/shared.scss
@@ -17,7 +17,9 @@
// fonticon links
a {
- .fab, .fas, .far {
+ .fab,
+ .fas,
+ .far {
color: var(--site-link-font-color);
}
}
@@ -25,3 +27,21 @@ a {
.page-title {
font-family: var(--site-font-fun-heading-family) !important;
}
+
+.subprocessors {
+ color: #130c49;
+ display: flex;
+ flex-direction: column;
+ align-items: flex-start;
+ gap: 20px;
+ width: 80%;
+ margin: auto;
+
+ @media screen and (max-width: 900px) {
+ width: 100%;
+
+ table {
+ width: 100%;
+ }
+ }
+}
diff --git a/src/pages/SLA.md b/src/pages/SLA.md
index 2a03e5193a..ae650bd40e 100644
--- a/src/pages/SLA.md
+++ b/src/pages/SLA.md
@@ -64,9 +64,9 @@ Planned maintenance windows are periods during which the availability of the ser
Weaviate employs different backup policies for each tier:
-- **Standard Tier:** Daily full backup stored in the same region for 7 days.
-- **Professional Tier:** Daily full backup stored in another region for 30 days.
-- **Business Critical Tier:** Daily full backup stored in another region for 30 days.
+- **Standard Tier:** Daily full backup, stored in the same region for 7 days.
+- **Professional Tier:** Daily full backup, stored in the same region for 14 days.
+- **Business Critical Tier:** Daily full backup, stored in the same region for 30 days.
### **Monitoring**
diff --git a/src/pages/_javascript.jsx b/src/pages/_javascript.jsx
new file mode 100644
index 0000000000..9621f4b06c
--- /dev/null
+++ b/src/pages/_javascript.jsx
@@ -0,0 +1,39 @@
+import React, { useEffect } from 'react';
+import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
+import Layout from '@theme/Layout';
+import { MetaSEO } from '/src/theme/MetaSEO';
+
+import Recipes from '/src/components/Javascript/Recipes';
+import Steps from '/src/components/Javascript/Steps';
+import Details from '/src/components/Javascript/Details';
+import Header from '/src/components/Javascript/Header';
+import List from '/src/components/Javascript/List';
+import Tabs from '/src/components/Javascript/Tabs';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+import Resources from '/src/components/Javascript/Resources';
+import BuildWith from '../components/Javascript/BWW';
+
+export default function Home() {
+ const { siteConfig } = useDocusaurusContext();
+
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/case-studies/_morningstarV2.jsx b/src/pages/case-studies/_morningstarV2.jsx
new file mode 100644
index 0000000000..9ed479bdd7
--- /dev/null
+++ b/src/pages/case-studies/_morningstarV2.jsx
@@ -0,0 +1,25 @@
+import React, { useState, useEffect } from 'react';
+import Layout from '@theme/Layout';
+import StudyHeader from '/src/components/CaseStudies/V2/Header';
+import ContactUsForm from '/src/components/ContactUsForm';
+import Main from '/src/components/CaseStudies/V2/Main';
+import Study from '/src/components/CaseStudies/V2/Study';
+import Stories from '/src/components/CaseStudies/V2/Stories';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+import Integrations from '/src/components/CaseStudies/V2/Integrations';
+
+export default function CaseStudyPage() {
+ return (
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/case-studies/finance.jsx b/src/pages/case-studies/finance.jsx
new file mode 100644
index 0000000000..a8f87b66db
--- /dev/null
+++ b/src/pages/case-studies/finance.jsx
@@ -0,0 +1,25 @@
+import React, { useState, useEffect } from 'react';
+import Layout from '@theme/Layout';
+import StudyHeader from '/src/components/Service/Finance/Header';
+import ContactUsForm from '/src/components/ContactUsForm';
+import Study from '/src/components/Service/Finance/Study';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+
+import Integrations from '/src/components/Service/CaseStudy/Integrations';
+
+export default function CaseStudyPage() {
+ return (
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/case-studies/index.jsx b/src/pages/case-studies/index.jsx
new file mode 100644
index 0000000000..ff04cc7a71
--- /dev/null
+++ b/src/pages/case-studies/index.jsx
@@ -0,0 +1,20 @@
+import React, { useState, useEffect } from 'react';
+import Layout from '@theme/Layout';
+import StudyHeader from '/src/components/CaseStudies/Header';
+import Main from '/src/components/CaseStudies/Main';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+
+export default function CaseStudyPage() {
+ return (
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/contact.jsx b/src/pages/contact.jsx
new file mode 100644
index 0000000000..4510f42862
--- /dev/null
+++ b/src/pages/contact.jsx
@@ -0,0 +1,25 @@
+import React from 'react';
+import Layout from '@theme/Layout';
+import { MetaSEO } from '/src/theme/MetaSEO';
+
+import Header from '/src/components/Contact/Header';
+import ContactFrom from '/src/components/Contact/contactForm.jsx';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+import CTA from '/src/components/Home/Redesign/CTA';
+
+export default function ContactPage() {
+ return (
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/demo.jsx b/src/pages/demo.jsx
new file mode 100644
index 0000000000..143f5ca67e
--- /dev/null
+++ b/src/pages/demo.jsx
@@ -0,0 +1,32 @@
+import React, { useState } from 'react';
+import Layout from '@theme/Layout';
+import Link from '@docusaurus/Link';
+import { MetaSEO } from '/src/theme/MetaSEO';
+import Head from '@docusaurus/Head';
+
+import Introduction from '/src/components/Signup/Demo/v2';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+
+export default function signUpPage() {
+ return (
+ <>
+
+ Request a Demo
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/ebooks/_index.jsx b/src/pages/ebooks/_index.jsx
new file mode 100644
index 0000000000..1793f7895a
--- /dev/null
+++ b/src/pages/ebooks/_index.jsx
@@ -0,0 +1,32 @@
+import React, { useState } from 'react';
+import Layout from '@theme/Layout';
+import Link from '@docusaurus/Link';
+import { MetaSEO } from '/src/theme/MetaSEO';
+import Head from '@docusaurus/Head';
+
+import Introduction from '/src/components/Signup/Guides';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+
+export default function signUpPage() {
+ return (
+ <>
+
+ Get Started For Free
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/ebooks/choosing-the-right-database-for-AI.jsx b/src/pages/ebooks/choosing-the-right-database-for-AI.jsx
new file mode 100644
index 0000000000..8b1b136336
--- /dev/null
+++ b/src/pages/ebooks/choosing-the-right-database-for-AI.jsx
@@ -0,0 +1,35 @@
+import React, { useState } from 'react';
+import Layout from '@theme/Layout';
+import Link from '@docusaurus/Link';
+import { MetaSEO } from '/src/theme/MetaSEO';
+import Head from '@docusaurus/Head';
+
+import Introduction from '/src/components/Signup/Ebooks/v2';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+
+export default function signUpPage() {
+ return (
+ <>
+
+ Ebook: Choosing the Right Database For AI
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/index.jsx b/src/pages/index.jsx
index ce3025b568..1600134f49 100644
--- a/src/pages/index.jsx
+++ b/src/pages/index.jsx
@@ -8,7 +8,7 @@ import HomepageHeader from '/src/components/Home/Redesign/Header';
import HomepageWhatYouCanDo from '/src/components/Home/Redesign/WhatYouCanDoUpdate';
import HomepageLovedByDevelopers from '/src/components/Home/Redesign/LovedByDeveloperUpdate/index.jsx';
import HomepageJoinCommunity from '/src/components/Home/Redesign/JoinCommunityUpdate';
-import HomepageIntegrations from '/src/components/Home/Redesign/IntegrationsUpdate';
+import HomepageIntegrations from '/src/components/Home/Redesign/IntegrationsUpdate/v2.jsx';
import HomepageTestimonials from '/src/components/Home/Redesign/Testimonials';
import Resources from '/src/components/Home/Redesign/Resources';
import CTA from '/src/components/Home/Redesign/CTA';
diff --git a/src/pages/learn/index.jsx b/src/pages/learn/index.jsx
new file mode 100644
index 0000000000..5c396f36f6
--- /dev/null
+++ b/src/pages/learn/index.jsx
@@ -0,0 +1,36 @@
+import React, { useEffect } from 'react';
+import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
+import Layout from '@theme/Layout';
+import { MetaSEO } from '/src/theme/MetaSEO';
+
+import Header from '/src/components/Learn/Header';
+import GetStarted from '/src/components/Learn/GetStarted';
+import Documentation from '/src/components/Learn/Documentation';
+import Examples from '/src/components/Learn/Examples';
+import Guides from '/src/components/Learn/Guides';
+import GoFurther from '/src/components/Learn/Further';
+import Resources from '../../components/Learn/Resources';
+import ThemeSwitch from '/src/components/ThemeSwitch';
+
+export default function Home() {
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/pages/learn/knowledgecards.jsx b/src/pages/learn/knowledgecards.jsx
index 648ca75543..ae69439706 100644
--- a/src/pages/learn/knowledgecards.jsx
+++ b/src/pages/learn/knowledgecards.jsx
@@ -1,7 +1,7 @@
import React, { useState } from 'react';
import Layout from '@theme/Layout';
import { MetaSEO } from '/src/theme/MetaSEO';
-
+import Head from '@docusaurus/Head';
import KnowledgeHeader from '/src/components/Knowledgebase/Knowledgeheader';
import KnowledgeBase from '/src/components/Knowledgebase/knowledgebase';
import ThemeSwitch from '/src/components/ThemeSwitch';
@@ -14,6 +14,9 @@ export default function KnowledgeBasePage() {
};
return (
+
+
+
+
);
diff --git a/src/pages/learn/knowledgecards/Knowledgeheader.jsx b/src/pages/learn/knowledgecards/Knowledgeheader.jsx
index dc54f31c30..3f43369768 100644
--- a/src/pages/learn/knowledgecards/Knowledgeheader.jsx
+++ b/src/pages/learn/knowledgecards/Knowledgeheader.jsx
@@ -6,9 +6,6 @@ import Head from '@docusaurus/Head';
export default function KnowledgeHeader() {
return (
<>
-
-
-
.default})
-
-*[Figure 1 - kNN search in a game of Boules.]*
-
-### A kNN search is computationally very expensive
-Comparing a search vector with 10, 100, or 1000 data vectors in just two dimensions is an easy job. But of course, in the real world, we are more likely to deal with millions (like in the Wikipedia dataset) or even billions of data items. In addition, the number of dimensions that most ML models use in semantic search goes up to hundreds or thousands of dimensions!
-
-The *brute* force of a **kNN search is computationally very expensive** - and depending on the size of your database, a single query could take anything from several seconds to even hours (yikes😅). If you compare a vector with 300 dimensions with 10M vectors, the vector search would need to do 300 x 10M = 3B computations! The number of required calculations increases linearly with the number of data points (O(n)) (Figure 2).
-
-.default})
-
-*[Figure 2 - O(n) and O(log n) complexity]*
-
-In summary, kNN search doesn't scale well, and it is hard to image using it with a large dataset in production.
-
-## The answer - Approximate nearest neighbors (ANN)
-Instead of comparing vectors one by one, vector databases use [Approximate Nearest Neighbor (ANN) algorithms](https://en.wikipedia.org/wiki/Nearest_neighbor_search#Approximation_methods), which trade off a bit of accuracy (hence the A in the name) for a huge gain in speed.
-
-ANN algorithms may not return the true k nearest vectors, but they are very efficient. ANN algorithms maintain good performance (sublinear time, e.g. (poly)logarithmic complexity, see Figure 2) on very large-scale datasets.
-
-*Note that most vector databases allow you to configure how your ANN algorithm should behave. This lets you find the right balance between the **recall tradeoff** (the fraction of results that are the true top-k nearest neighbors), **latency**, **throughput** (queries per second) and **import time**.*.default})
-
-*[Figure 4 - Proximity graph-based ANN search]*
-
-Check out [Weaviate ANN benchmarks](/developers/weaviate/benchmarks/ann) to see how HNSW performed on realistic large-scale datasets. You can use it to compare the tradeoffs between recall, QPS, latency, and import time.
+
+
+
+
+
+Weaviate output:
+
+```text
+Interstellar 2014 157336
+Distance to query: 0.354
+
+Gravity 2013 49047
+Distance to query: 0.384
+
+Arrival 2016 329865
+Distance to query: 0.386
+
+Armageddon 1998 95
+Distance to query: 0.400
+
+Godzilla 1998 929
+Distance to query: 0.441
+```
+
+
+
+
+
+
+
+Weaviate output:
+
+```text
+Deadpool 2 2018 383498
+Distance to query: 0.670
+
+Bloodshot 2020 338762
+Distance to query: 0.677
+
+Deadpool 2016 293660
+Distance to query: 0.678
+
+300 2007 1271
+Distance to query: 0.682
+
+The Hunt for Red October 1990 1669
+Distance to query: 0.683
+```
+
+