From 4e723ebe295942a2e4e4d7b227e548d753c4beab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jacques=20Verr=C3=A9?= <jverre@gmail.com>
Date: Mon, 30 Dec 2024 17:25:54 +0100
Subject: [PATCH] Added support for llms.txt and llms-full.txt

---
 .../documentation/docs/changelog.md           |  25 ++++-
 .../documentation/docs/cookbook/aisuite.ipynb |   9 ++
 .../docs/cookbook/anthropic.ipynb             |   9 ++
 .../documentation/docs/cookbook/bedrock.ipynb |   4 +
 .../evaluate_hallucination_metric.ipynb       |  89 +++------------
 .../cookbook/evaluate_moderation_metric.ipynb | 102 +++---------------
 .../documentation/docs/cookbook/gemini.ipynb  |   4 +
 .../documentation/docs/cookbook/groq.ipynb    |   4 +
 .../docs/cookbook/haystack.ipynb              |  41 ++-----
 .../docs/cookbook/langchain.ipynb             |   4 +
 .../docs/cookbook/langgraph.ipynb             |   4 +
 .../documentation/docs/cookbook/litellm.ipynb |   4 +
 .../docs/cookbook/llama-index.ipynb           |   4 +
 .../documentation/docs/cookbook/ollama.ipynb  |   4 +
 .../documentation/docs/cookbook/openai.ipynb  |   4 +
 .../docs/cookbook/predibase.ipynb             |   4 +
 .../docs/cookbook/quickstart_notebook.ipynb   |   4 +
 .../documentation/docs/cookbook/ragas.ipynb   |   4 +
 .../documentation/docs/cookbook/watsonx.ipynb |   4 +
 .../documentation/docs/evaluation/concepts.md |   1 +
 .../docs/evaluation/evaluate_your_llm.md      |   1 +
 .../docs/evaluation/manage_datasets.md        |   1 +
 .../evaluation/metrics/answer_relevance.md    |   1 +
 .../evaluation/metrics/context_precision.md   |   1 +
 .../docs/evaluation/metrics/context_recall.md |   1 +
 .../docs/evaluation/metrics/custom_metric.md  |   1 +
 .../docs/evaluation/metrics/custom_model.md   |   1 +
 .../docs/evaluation/metrics/g_eval.md         |   1 +
 .../docs/evaluation/metrics/hallucination.md  |   1 +
 .../evaluation/metrics/heuristic_metrics.md   |   1 +
 .../docs/evaluation/metrics/moderation.md     |   1 +
 .../docs/evaluation/metrics/overview.md       |   1 +
 .../docs/evaluation/playground.md             |   1 +
 .../evaluation/update_existing_experiment.md  |   1 +
 .../documentation/docs/faq.md                 |   2 +-
 .../documentation/docs/home.md                |   2 +-
 .../docs/library/managing_prompts_in_code.mdx |   1 +
 .../docs/library/prompt_management.mdx        |   1 +
 .../documentation/docs/production/gateway.mdx |   1 +
 .../documentation/docs/quickstart.mdx         |   2 +-
 .../documentation/docs/roadmap.md             |   2 +-
 .../configure/anonymous_usage_statistics.md   |   1 +
 .../docs/self-host/kubernetes.md              |   1 +
 .../docs/self-host/local_deployment.md        |   1 +
 .../documentation/docs/self-host/overview.md  |   1 +
 .../docs/testing/pytest_integration.md        |   2 +-
 .../docs/tracing/annotate_traces.md           |   2 +-
 .../docs/tracing/cost_tracking.md             |   5 +
 .../documentation/docs/tracing/export_data.md |   1 +
 .../docs/tracing/integrations/aisuite.md      |   1 +
 .../docs/tracing/integrations/anthropic.md    |   1 +
 .../docs/tracing/integrations/bedrock.md      |   1 +
 .../docs/tracing/integrations/dify.mdx        |   5 +-
 .../docs/tracing/integrations/gemini.md       |   1 +
 .../docs/tracing/integrations/groq.md         |   1 +
 .../docs/tracing/integrations/haystack.md     |   1 +
 .../docs/tracing/integrations/langchain.md    |   1 +
 .../docs/tracing/integrations/langgraph.md    |   1 +
 .../docs/tracing/integrations/litellm.md      |   1 +
 .../docs/tracing/integrations/llama_index.md  |   1 +
 .../docs/tracing/integrations/ollama.md       |   1 +
 .../docs/tracing/integrations/openai.md       |   1 +
 .../docs/tracing/integrations/overview.md     |   2 +-
 .../docs/tracing/integrations/predibase.md    |   1 +
 .../docs/tracing/integrations/ragas.md        |   1 +
 .../docs/tracing/integrations/watsonx.md      |   1 +
 .../documentation/docs/tracing/log_agents.mdx |   1 +
 .../docs/tracing/log_distributed_traces.md    |   2 +-
 .../docs/tracing/log_multimodal_traces.md     |   2 +-
 .../documentation/docs/tracing/log_traces.mdx |  13 ++-
 .../docs/tracing/production_monitoring.md     |   4 +-
 .../docs/tracing/sdk_configuration.mdx        |   5 +-
 .../documentation/docusaurus.config.ts        |  13 ++-
 .../documentation/package-lock.json           |  18 ++++
 .../documentation/package.json                |   1 +
 .../documentation/sidebars.ts                 |   2 +-
 76 files changed, 235 insertions(+), 215 deletions(-)

diff --git a/apps/opik-documentation/documentation/docs/changelog.md b/apps/opik-documentation/documentation/docs/changelog.md
index 4a3d1d7242..c3cd372b11 100644
--- a/apps/opik-documentation/documentation/docs/changelog.md
+++ b/apps/opik-documentation/documentation/docs/changelog.md
@@ -1,10 +1,33 @@
 ---
-sidebar_position: 101
 sidebar_label: Changelog
+description: Weelkly changelog for Opik
 ---
 
 # Weekly Changelog
 
+## Week of 2024-12-30
+
+**Opik Dashboard**:
+
+- Added duration chart to the project dashboard
+- Prompt metadata can now be set and viewed in the UI, this can be used to store any additional information about the prompt
+- Playground prompts and settings are now cached when you navigate away from the page
+
+**SDK**:
+
+- Introduced a new `OPIK_TRACK_DISABLE` environment variable to disable the tracking of traces and spans
+- We now log usage information for traces logged using the LlamaIndex integration
+
+## Week of 2024-12-23
+
+**SDK**:
+
+- Improved error messages when getting a rate limit when using the `evaluate` method
+- Added support for a new metadata field in the `Prompt` object, this field is used to store any additional information about the prompt.
+- Updated the library used to create uuidv7 IDs
+- New Guardrails integration
+- New DSPY integration
+
 ## Week of 2024-12-16
 
 **Opik Dashboard**:
diff --git a/apps/opik-documentation/documentation/docs/cookbook/aisuite.ipynb b/apps/opik-documentation/documentation/docs/cookbook/aisuite.ipynb
index f1384553d3..6c3530c2c4 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/aisuite.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/aisuite.ipynb
@@ -1,5 +1,14 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the aisuite Python SDK\n",
+    "---"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
diff --git a/apps/opik-documentation/documentation/docs/cookbook/anthropic.ipynb b/apps/opik-documentation/documentation/docs/cookbook/anthropic.ipynb
index 8a4f381021..da705cdd25 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/anthropic.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/anthropic.ipynb
@@ -1,5 +1,14 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the Anthropic Python SDK\n",
+    "---"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
diff --git a/apps/opik-documentation/documentation/docs/cookbook/bedrock.ipynb b/apps/opik-documentation/documentation/docs/cookbook/bedrock.ipynb
index 4392736a88..a86ed6e259 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/bedrock.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/bedrock.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with AWS Bedrock\n",
+    "---\n",
+    "\n",
     "# Using Opik with AWS Bedrock\n",
     "\n",
     "Opik integrates with AWS Bedrock to provide a simple way to log traces for all Bedrock LLM calls. This works for all supported models, including if you are using the streaming API.\n"
diff --git a/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb b/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb
index ca197ec3d0..29e9ed3e49 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb
@@ -1,5 +1,15 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "sidebar_label: Evaluating Opik's Hallucination Metric\n",
+    "description: Cookbook that evalutes Opik's Hallucination Metric, showcasing both how to use the `evaluation` functionality in the platform as well as the quality of the Hallucination metric included in the SDK. It is a complex example that doesn't always align with how the `evaluate` function works.\n",
+    "---"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -51,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -73,17 +83,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "OPIK: Created a \"HaluEval\" dataset at https://www.comet.com/opik/jacques-comet/redirect/datasets?name=HaluEval.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Create dataset\n",
     "import opik\n",
@@ -128,70 +130,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Evaluation: 100%|██████████| 50/50 [00:07<00:00,  6.37it/s]\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">╭─ HaluEval (50 samples) ───────────────────╮\n",
-       "│                                           │\n",
-       "│ <span style=\"font-weight: bold\">Total time:       </span> 00:00:08               │\n",
-       "│ <span style=\"font-weight: bold\">Number of samples:</span> 50                     │\n",
-       "│                                           │\n",
-       "│ <span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Correct hallucination score: 0.8000 (avg)</span> │\n",
-       "│                                           │\n",
-       "╰───────────────────────────────────────────╯\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "╭─ HaluEval (50 samples) ───────────────────╮\n",
-       "│                                           │\n",
-       "│ \u001b[1mTotal time:       \u001b[0m 00:00:08               │\n",
-       "│ \u001b[1mNumber of samples:\u001b[0m 50                     │\n",
-       "│                                           │\n",
-       "│ \u001b[1;32mCorrect hallucination score: 0.8000 (avg)\u001b[0m │\n",
-       "│                                           │\n",
-       "╰───────────────────────────────────────────╯\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Uploading results to Opik <span style=\"color: #808000; text-decoration-color: #808000\">...</span> \n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Uploading results to Opik \u001b[33m...\u001b[0m \n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">View the results <a href=\"https://www.comet.com/opik/jacques-comet/experiments/0193aaa8-7d37-7512-8450-0ed502c074e4/compare?experiments=%5B%2206756b2b-4988-7701-8000-8c2b063aa4dc%22%5D\" target=\"_blank\">in your Opik dashboard</a>.\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "View the results \u001b]8;id=436842;https://www.comet.com/opik/jacques-comet/experiments/0193aaa8-7d37-7512-8450-0ed502c074e4/compare?experiments=%5B%2206756b2b-4988-7701-8000-8c2b063aa4dc%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "from opik.evaluation.metrics import Hallucination, Equals\n",
     "from opik.evaluation import evaluate\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb b/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb
index c70540e095..dd2296d696 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb
@@ -4,6 +4,11 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "sidebar_label: Evaluate Opik's Moderation Metric\n",
+    "description: Cookbook that evalutes Opik's Moderation Metric, showcasing both how to use the `evaluation` functionality in the platform as well as the quality of the Modelration metric included in the SDK. It is a complex example that doesn't always align with how the `evaluate` function works.\n",
+    "---\n",
+    "\n",
     "# Evaluating Opik's Moderation Metric\n",
     "\n",
     "For this guide we will be evaluating the Moderation metric included in the LLM Evaluation SDK which will showcase both how to use the `evaluation` functionality in the platform as well as the quality of the Moderation metric included in the SDK."
@@ -22,35 +27,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Note: you may need to restart the kernel to use updated packages.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%pip install --upgrade --quiet opik pandas"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "OPIK: Your Opik API key is available in your account settings, can be found at https://www.comet.com/api/my/settings/ for Opik cloud\n",
-      "OPIK: Configuration saved to file: /Users/jacquesverre/.opik.config\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import opik\n",
     "\n",
@@ -68,7 +56,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -90,7 +78,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -149,73 +137,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Evaluation:   0%|          | 0/50 [00:00<?, ?it/s]OPIK: Started logging traces to the \"Default Project\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.\n",
-      "OPIK: Started logging traces to the \"Default Project\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.\n",
-      "OPIK: Started logging traces to the \"Default Project\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.\n",
-      "Evaluation: 100%|██████████| 50/50 [00:05<00:00,  8.86it/s]\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">╭─ OpenAIModerationDataset (50 samples) ─╮\n",
-       "│                                        │\n",
-       "│ <span style=\"font-weight: bold\">Total time:       </span> 00:00:06            │\n",
-       "│ <span style=\"font-weight: bold\">Number of samples:</span> 50                  │\n",
-       "│                                        │\n",
-       "│ <span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Correct moderation score: 0.8600 (avg)</span> │\n",
-       "│                                        │\n",
-       "╰────────────────────────────────────────╯\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "╭─ OpenAIModerationDataset (50 samples) ─╮\n",
-       "│                                        │\n",
-       "│ \u001b[1mTotal time:       \u001b[0m 00:00:06            │\n",
-       "│ \u001b[1mNumber of samples:\u001b[0m 50                  │\n",
-       "│                                        │\n",
-       "│ \u001b[1;32mCorrect moderation score: 0.8600 (avg)\u001b[0m │\n",
-       "│                                        │\n",
-       "╰────────────────────────────────────────╯\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Uploading results to Opik <span style=\"color: #808000; text-decoration-color: #808000\">...</span> \n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "Uploading results to Opik \u001b[33m...\u001b[0m \n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">View the results <a href=\"https://www.comet.com/opik/jacques-comet/experiments/01939721-3a29-7f9a-ac8b-737923e30e31/compare?experiments=%5B%2206766a9d-2e22-7b5a-8000-afa08cbccbb4%22%5D\" target=\"_blank\">in your Opik dashboard</a>.\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "View the results \u001b]8;id=373788;https://www.comet.com/opik/jacques-comet/experiments/01939721-3a29-7f9a-ac8b-737923e30e31/compare?experiments=%5B%2206766a9d-2e22-7b5a-8000-afa08cbccbb4%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "from opik.evaluation.metrics import Moderation, Equals\n",
     "from opik.evaluation import evaluate\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/gemini.ipynb b/apps/opik-documentation/documentation/docs/cookbook/gemini.ipynb
index 849d435d13..670494b173 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/gemini.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/gemini.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the Gemini Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with Gemini\n",
     "\n",
     "Opik integrates with Gemini to provide a simple way to log traces for all Gemini LLM calls. This works for all Gemini models."
diff --git a/apps/opik-documentation/documentation/docs/cookbook/groq.ipynb b/apps/opik-documentation/documentation/docs/cookbook/groq.ipynb
index c1e1e827ed..7451394080 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/groq.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/groq.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with Groq\n",
+    "---\n",
+    "\n",
     "# Using Opik with Groq\n",
     "\n",
     "Opik integrates with Groq to provide a simple way to log traces for all Groq LLM calls. This works for all Groq models."
diff --git a/apps/opik-documentation/documentation/docs/cookbook/haystack.ipynb b/apps/opik-documentation/documentation/docs/cookbook/haystack.ipynb
index 73cfd2fe8d..4fed419b58 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/haystack.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/haystack.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with Haystack\n",
+    "---\n",
+    "\n",
     "# Using Opik with Haystack\n",
     "\n",
     "[Haystack](https://docs.haystack.deepset.ai/docs/intro) is an open-source framework for building production-ready LLM applications, retrieval-augmented generative pipelines and state-of-the-art search systems that work intelligently over large document collections.\n",
@@ -44,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -72,26 +76,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "OPIK: Traces will not be logged to Opik because Haystack tracing is disabled. To enable, set the HAYSTACK_CONTENT_TRACING_ENABLED environment variable to true before importing Haystack.\n",
-      "OPIK: Started logging traces to the \"Default Project\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Trace ID: 0675969a-fd40-7fcb-8000-0287dec9559f\n",
-      "ChatMessage(content='Berlin ist die Hauptstadt Deutschlands und zugleich eine der aufregendsten und vielfältigsten Städte Europas. Die Metropole hat eine reiche Geschichte, die sich in ihren historischen Gebäuden, Museen und Denkmälern widerspiegelt. Berlin ist auch bekannt für seine lebendige Kunst- und Kulturszene, mit unzähligen Galerien, Theatern und Musikveranstaltungen.\\n\\nDie Stadt ist zudem ein Schmelztiegel der Kulturen, was sich in der vielfältigen Gastronomie, den lebhaften Märkten und den multikulturellen Vierteln widerspiegelt. Berlin bietet auch eine lebendige Nightlife-Szene, mit zahlreichen Bars, Clubs und Veranstaltungen für jeden Geschmack.\\n\\nNeben all dem kulturellen Reichtum hat Berlin auch eine grüne Seite, mit vielen Parks, Gärten und Seen, die zum Entspannen und Erholen einladen. Insgesamt ist Berlin eine Stadt, die für jeden etwas zu bieten hat und die Besucher mit ihrer Vielfalt und Offenheit begeistert.', role=<ChatRole.ASSISTANT: 'assistant'>, name=None, meta={'model': 'gpt-3.5-turbo-0125', 'index': 0, 'finish_reason': 'stop', 'usage': {'completion_tokens': 255, 'prompt_tokens': 29, 'total_tokens': 284, 'completion_tokens_details': CompletionTokensDetails(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0), 'prompt_tokens_details': PromptTokensDetails(audio_tokens=0, cached_tokens=0)}})\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import os\n",
     "\n",
@@ -161,7 +148,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -181,17 +168,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Trace ID: 067596ab-da00-7c1f-8000-53f7af5fc3de\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "response = pipe.run(\n",
     "    data={\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb b/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb
index d87979ea2e..511e83cc90 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the LangChain Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with Langchain\n",
     "\n",
     "For this guide, we will be performing a text to sql query generation task using LangChain. We will be using the Chinook database which contains the SQLite database of a music store with both employee, customer and invoice data.\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/langgraph.ipynb b/apps/opik-documentation/documentation/docs/cookbook/langgraph.ipynb
index ad72214b35..cb074f5c67 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/langgraph.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/langgraph.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the LangGraph Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with LangGraph\n",
     "\n",
     "This notebook showcases how to use Opik with LangGraph. [LangGraph](https://langchain-ai.github.io/langgraph/) is a library for building stateful, multi-actor applications with LLMs, used to create agent and multi-agent workflows\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/litellm.ipynb b/apps/opik-documentation/documentation/docs/cookbook/litellm.ipynb
index 2fbcf49d98..41d37d1ac0 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/litellm.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/litellm.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the LiteLLM Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with LiteLLM\n",
     "\n",
     "Lite allows you to call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]. You can learn more about LiteLLM [here](https://github.com/BerriAI/litellm).\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/llama-index.ipynb b/apps/opik-documentation/documentation/docs/cookbook/llama-index.ipynb
index ede2541e32..8f8729f956 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/llama-index.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/llama-index.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the LlamaIndex Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with LlamaIndex\n",
     "\n",
     "This notebook showcases how to use Opik with LlamaIndex. [LlamaIndex](https://github.com/run-llama/llama_index) is a flexible data framework for building LLM applications:\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/ollama.ipynb b/apps/opik-documentation/documentation/docs/cookbook/ollama.ipynb
index 3843d454cf..b029ac6959 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/ollama.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/ollama.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the Ollama Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with Ollama\n",
     "\n",
     "[Ollama](https://ollama.com/) allows users to run, interact with, and deploy AI models locally on their machines without the need for complex infrastructure or cloud dependencies.\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/openai.ipynb b/apps/opik-documentation/documentation/docs/cookbook/openai.ipynb
index 1e0e08f498..d20ac083ba 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/openai.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/openai.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the OpenAI Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with OpenAI\n",
     "\n",
     "Opik integrates with OpenAI to provide a simple way to log traces for all OpenAI LLM calls. This works for all OpenAI models, including if you are using the streaming API.\n"
diff --git a/apps/opik-documentation/documentation/docs/cookbook/predibase.ipynb b/apps/opik-documentation/documentation/docs/cookbook/predibase.ipynb
index ace1e98158..68b3fab691 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/predibase.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/predibase.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with Predibase\n",
+    "---\n",
+    "\n",
     "# Using Opik with Predibase\n",
     "\n",
     "This notebook demonstrates how to use Predibase as an LLM provider with LangChain, and how to integrate Opik for tracking and logging.\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb b/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb
index 8f9ac6ec92..d8d928421b 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Quickstart cookbook that showcases Opik's evaluation, tracing and prompt management functionality.\n",
+    "---\n",
+    "\n",
     "# Quickstart notebook - Summarization task\n",
     "\n",
     "In this notebook, we will look at how you can use Opik to track your LLM calls, chains and agents. We will introduce the concept of tracing and how to automate the evaluation of your LLM workflows.\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/ragas.ipynb b/apps/opik-documentation/documentation/docs/cookbook/ragas.ipynb
index e33acdd7b6..95e6b44edc 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/ragas.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/ragas.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with the Ragas Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Ragas to evaluate RAG pipelines\n",
     "\n",
     "In this notebook, we will showcase how to use Opik with Ragas for monitoring and evaluation of RAG (Retrieval-Augmented Generation) pipelines.\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/watsonx.ipynb b/apps/opik-documentation/documentation/docs/cookbook/watsonx.ipynb
index 339ac13e0e..4d241a660f 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/watsonx.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/watsonx.ipynb
@@ -4,6 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "---\n",
+    "description: Cookbook that showcases Opik's integration with Watsonx through the LiteLLM Python SDK\n",
+    "---\n",
+    "\n",
     "# Using Opik with watsonx\n",
     "\n",
     "Opik integrates with watsonx to provide a simple way to log traces for all watsonx LLM calls. This works for all watsonx models."
diff --git a/apps/opik-documentation/documentation/docs/evaluation/concepts.md b/apps/opik-documentation/documentation/docs/evaluation/concepts.md
index 71c2492e56..fe2bfc5ef2 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/concepts.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/concepts.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Concepts
+description: Introduces the concepts behind Opik's evaluation framework
 ---
 
 # Evaluation Concepts
diff --git a/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md b/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md
index 0ce5e6eb3b..7504a700df 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Evaluate your LLM Application
+description: Step by step guide on how to evaluate your LLM application
 ---
 
 # Evaluate your LLM Application
diff --git a/apps/opik-documentation/documentation/docs/evaluation/manage_datasets.md b/apps/opik-documentation/documentation/docs/evaluation/manage_datasets.md
index b88119f546..8e53f46e4f 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/manage_datasets.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/manage_datasets.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Manage Datasets
+description: Guides you through the process of creating and managing datasets
 ---
 
 # Manage Datasets
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/answer_relevance.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/answer_relevance.md
index 9202ba5b14..632f0fb073 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/answer_relevance.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/answer_relevance.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: AnswerRelevance
+description: Describes the Answer Relevance metric
 ---
 
 # Answer Relevance
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/context_precision.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/context_precision.md
index 64a8a4803d..6088b49a79 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/context_precision.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/context_precision.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: ContextPrecision
+description: Describes the Context Precision metric
 ---
 
 # ContextPrecision
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/context_recall.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/context_recall.md
index 62d1977666..7ffc8e23d6 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/context_recall.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/context_recall.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: ContextRecall
+description: Describes the Context Recall metric
 ---
 
 # ContextRecall
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_metric.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_metric.md
index 61dd84aac3..f60044e959 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_metric.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_metric.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Custom Metric
+description: Describes how to create your own metric to use with Opik's evaluation framework
 toc_max_heading_level: 4
 ---
 
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_model.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_model.md
index 932fab869c..cd14f72de7 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_model.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/custom_model.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Customize models for LLM as a Judge metrics
+description: Describes how to use a custom model for Opik's built-in LLM as a Judge metrics
 toc_max_heading_level: 4
 ---
 
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/g_eval.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/g_eval.md
index 01234d0897..08af86405c 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/g_eval.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/g_eval.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: G-Eval
+description: Describes Opik's built-in G-Eval metric which is is a task agnostic LLM as a Judge metric
 ---
 
 # G-Eval
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/hallucination.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/hallucination.md
index f21bd74e49..827d50ce64 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/hallucination.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/hallucination.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Hallucination
+description: Describes the Hallucination metric
 ---
 
 # Hallucination
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/heuristic_metrics.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/heuristic_metrics.md
index 43d4e7c257..4ef08243f5 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/heuristic_metrics.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/heuristic_metrics.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Heuristic Metrics
+description: Describes all the built-in heuristic metrics provided by Opik
 ---
 
 # Heuristic Metrics
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/moderation.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/moderation.md
index 0ea20a31de..ba5dd4378c 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/moderation.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/moderation.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Moderation
+description: Describes the Moderation metric
 ---
 
 # Moderation
diff --git a/apps/opik-documentation/documentation/docs/evaluation/metrics/overview.md b/apps/opik-documentation/documentation/docs/evaluation/metrics/overview.md
index aae67ca959..d8f39b3a0b 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/metrics/overview.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/metrics/overview.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Overview
+description: Describes all the built-in evaluation metrics provided by Opik
 ---
 
 # Overview
diff --git a/apps/opik-documentation/documentation/docs/evaluation/playground.md b/apps/opik-documentation/documentation/docs/evaluation/playground.md
index 563dc76006..f23928b6eb 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/playground.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/playground.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Prompt playground
+description: Describes Opik's prompt playground that can be used to quickly try out different prompts
 ---
 
 # Prompt Playground
diff --git a/apps/opik-documentation/documentation/docs/evaluation/update_existing_experiment.md b/apps/opik-documentation/documentation/docs/evaluation/update_existing_experiment.md
index 5c3fc34f28..b7baf41f1a 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/update_existing_experiment.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/update_existing_experiment.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Update an existing experiment
+description: Guides you through the process of updating an existing experiment
 ---
 
 # Update an existing experiment
diff --git a/apps/opik-documentation/documentation/docs/faq.md b/apps/opik-documentation/documentation/docs/faq.md
index d6d8575397..ce5f840ad9 100644
--- a/apps/opik-documentation/documentation/docs/faq.md
+++ b/apps/opik-documentation/documentation/docs/faq.md
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 102
 sidebar_label: FAQ
+description: Frequently Asked Questions
 ---
 
 # FAQ
diff --git a/apps/opik-documentation/documentation/docs/home.md b/apps/opik-documentation/documentation/docs/home.md
index f3d70ca19d..324f77c160 100644
--- a/apps/opik-documentation/documentation/docs/home.md
+++ b/apps/opik-documentation/documentation/docs/home.md
@@ -1,7 +1,7 @@
 ---
-sidebar_position: 1
 slug: /
 sidebar_label: Home
+description: Opik documentation home page
 ---
 
 # Opik by Comet
diff --git a/apps/opik-documentation/documentation/docs/library/managing_prompts_in_code.mdx b/apps/opik-documentation/documentation/docs/library/managing_prompts_in_code.mdx
index 5bf078ed8a..5a49355f7e 100644
--- a/apps/opik-documentation/documentation/docs/library/managing_prompts_in_code.mdx
+++ b/apps/opik-documentation/documentation/docs/library/managing_prompts_in_code.mdx
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Versioning prompts stored in code
+description: Describes how to version prompts stored in code
 ---
 
 import Tabs from "@theme/Tabs";
diff --git a/apps/opik-documentation/documentation/docs/library/prompt_management.mdx b/apps/opik-documentation/documentation/docs/library/prompt_management.mdx
index 114c96e79e..095ceb6a69 100644
--- a/apps/opik-documentation/documentation/docs/library/prompt_management.mdx
+++ b/apps/opik-documentation/documentation/docs/library/prompt_management.mdx
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Overview
+description: Describes how to manage prompts in Opik
 ---
 
 import Tabs from "@theme/Tabs";
diff --git a/apps/opik-documentation/documentation/docs/production/gateway.mdx b/apps/opik-documentation/documentation/docs/production/gateway.mdx
index de2ed20249..8b7eddb58f 100644
--- a/apps/opik-documentation/documentation/docs/production/gateway.mdx
+++ b/apps/opik-documentation/documentation/docs/production/gateway.mdx
@@ -1,5 +1,6 @@
 ---
 sidebar_label: LLM Gateway
+description: Describes how to use the Opik LLM gateway and how to integrate with the Kong AI Gateway
 ---
 
 import Tabs from "@theme/Tabs";
diff --git a/apps/opik-documentation/documentation/docs/quickstart.mdx b/apps/opik-documentation/documentation/docs/quickstart.mdx
index c5774030a1..0a2675b241 100644
--- a/apps/opik-documentation/documentation/docs/quickstart.mdx
+++ b/apps/opik-documentation/documentation/docs/quickstart.mdx
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 2
 sidebar_label: Quickstart
+description: This guide helps you integrate the Opik platform with your existing LLM application
 ---
 
 import Tabs from "@theme/Tabs";
diff --git a/apps/opik-documentation/documentation/docs/roadmap.md b/apps/opik-documentation/documentation/docs/roadmap.md
index 93cbbcd521..3f04f58bb0 100644
--- a/apps/opik-documentation/documentation/docs/roadmap.md
+++ b/apps/opik-documentation/documentation/docs/roadmap.md
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 100
 sidebar_label: Roadmap
+description: Opik Roadmap
 ---
 
 # Roadmap
diff --git a/apps/opik-documentation/documentation/docs/self-host/configure/anonymous_usage_statistics.md b/apps/opik-documentation/documentation/docs/self-host/configure/anonymous_usage_statistics.md
index b6f1826b63..9ddab38eb5 100644
--- a/apps/opik-documentation/documentation/docs/self-host/configure/anonymous_usage_statistics.md
+++ b/apps/opik-documentation/documentation/docs/self-host/configure/anonymous_usage_statistics.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Anonymous Usage Statistics
+description: Describes the usage statistics that are collected by Opik
 ---
 
 # Anonymous Usage Statistics
diff --git a/apps/opik-documentation/documentation/docs/self-host/kubernetes.md b/apps/opik-documentation/documentation/docs/self-host/kubernetes.md
index 9ba2055038..e91b6e76f6 100644
--- a/apps/opik-documentation/documentation/docs/self-host/kubernetes.md
+++ b/apps/opik-documentation/documentation/docs/self-host/kubernetes.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Production (Kubernetes)
+description: Describes how to run Opik on a Kubernetes cluster
 ---
 
 # Production ready Kubernetes deployment
diff --git a/apps/opik-documentation/documentation/docs/self-host/local_deployment.md b/apps/opik-documentation/documentation/docs/self-host/local_deployment.md
index 1ef973dabf..7225ab2de5 100644
--- a/apps/opik-documentation/documentation/docs/self-host/local_deployment.md
+++ b/apps/opik-documentation/documentation/docs/self-host/local_deployment.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Local (Docker Compose)
+description: Describes how to run Opik locally using Docker Compose
 ---
 
 # Local Deployments using Docker Compose
diff --git a/apps/opik-documentation/documentation/docs/self-host/overview.md b/apps/opik-documentation/documentation/docs/self-host/overview.md
index f2a6e2405d..bd868d74f7 100644
--- a/apps/opik-documentation/documentation/docs/self-host/overview.md
+++ b/apps/opik-documentation/documentation/docs/self-host/overview.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Overview
+description: High-level overview on how to self-host Opik
 ---
 
 # Self-hosting Opik
diff --git a/apps/opik-documentation/documentation/docs/testing/pytest_integration.md b/apps/opik-documentation/documentation/docs/testing/pytest_integration.md
index 4fd1decbfe..fbd8b65000 100644
--- a/apps/opik-documentation/documentation/docs/testing/pytest_integration.md
+++ b/apps/opik-documentation/documentation/docs/testing/pytest_integration.md
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 1
 sidebar_label: Pytest Integration
+description: Describes how to use Opik with Pytest to write LLM unit tests
 ---
 
 # Pytest Integration
diff --git a/apps/opik-documentation/documentation/docs/tracing/annotate_traces.md b/apps/opik-documentation/documentation/docs/tracing/annotate_traces.md
index b840a6566a..97bed1e518 100644
--- a/apps/opik-documentation/documentation/docs/tracing/annotate_traces.md
+++ b/apps/opik-documentation/documentation/docs/tracing/annotate_traces.md
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 4
 sidebar_label: Annotate Traces
+description: Describes how to annotate traces using the Opik SDK and UI
 ---
 
 # Annotate Traces
diff --git a/apps/opik-documentation/documentation/docs/tracing/cost_tracking.md b/apps/opik-documentation/documentation/docs/tracing/cost_tracking.md
index 0fbc741e78..031a4a92cd 100644
--- a/apps/opik-documentation/documentation/docs/tracing/cost_tracking.md
+++ b/apps/opik-documentation/documentation/docs/tracing/cost_tracking.md
@@ -1,3 +1,8 @@
+---
+sidebar_label: Cost Tracking
+description: Describes how to track and monitor costs for your LLM applications using Opik
+---
+
 # Cost Tracking
 
 Opik has been designed to track and monitor costs for your LLM applications by measuring token usage across all traces. Using the Opik dashboard, you can analyze spending patterns and quickly identify cost anomalies. All costs across Opik are estimated and displayed in USD.
diff --git a/apps/opik-documentation/documentation/docs/tracing/export_data.md b/apps/opik-documentation/documentation/docs/tracing/export_data.md
index 665c4b9a5d..5a2c322142 100644
--- a/apps/opik-documentation/documentation/docs/tracing/export_data.md
+++ b/apps/opik-documentation/documentation/docs/tracing/export_data.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Export Traces and Spans
+description: Describes how to export traces and spans from the Opik platform.
 toc_max_heading_level: 4
 ---
 
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/aisuite.md b/apps/opik-documentation/documentation/docs/tracing/integrations/aisuite.md
index b141afe642..422568f131 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/aisuite.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/aisuite.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: aisuite
+description: Describes how to track aisuite LLM calls using Opik
 ---
 
 # aisuite
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/anthropic.md b/apps/opik-documentation/documentation/docs/tracing/integrations/anthropic.md
index 5ee17f2429..eae608dffa 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/anthropic.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/anthropic.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Anthropic
+description: Describes how to track Anthropic LLM calls using Opik
 ---
 
 # Anthropic
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/bedrock.md b/apps/opik-documentation/documentation/docs/tracing/integrations/bedrock.md
index dbd21a76ab..264166cc7c 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/bedrock.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/bedrock.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Bedrock
+description: Describes how to track Bedrock LLM calls using Opik
 ---
 
 # AWS Bedrock
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/dify.mdx b/apps/opik-documentation/documentation/docs/tracing/integrations/dify.mdx
index a1ad6a1995..4467d355f8 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/dify.mdx
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/dify.mdx
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Dify
+description: Describes how to use Opik with Dify
 ---
 
 import Tabs from "@theme/Tabs";
@@ -25,6 +26,7 @@ Follow these simple steps to connect Dify with Opik:
 <TabItem value="Hosted Opik" title="Hosted Opik">
 
 Fill in these fields:
+
 - **API Key**: Your Comet API Key
 - **Project**: Your preferred project name (if left empty, it will be created automatically)
 - **Workspace**: Your Comet Workspace name (must already exist)
@@ -35,6 +37,7 @@ Fill in these fields:
 <TabItem value="Self-Hosted Opik" title="Self-Hosted Opik">
 
 Fill in these fields:
+
 - **API Key**: Leave this empty
 - **Project**: Your preferred project name (if left empty, it will be created automatically)
 - **Workspace**: Type `default`
@@ -52,4 +55,4 @@ After setup, you can view your application traces by:
 2. Finding and clicking the **OPIK** button in the top-right corner
 3. Selecting **View** to open your Opik project dashboard
 
-![How to view your Opik project](/img/tracing/dify_view_project.png)
\ No newline at end of file
+![How to view your Opik project](/img/tracing/dify_view_project.png)
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/gemini.md b/apps/opik-documentation/documentation/docs/tracing/integrations/gemini.md
index 797e9e3795..e4392bb54d 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/gemini.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/gemini.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Gemini - Google AI Studio
+description: Describes how to track Gemini LLM calls using Opik
 ---
 
 # Gemini - Google AI Studio
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/groq.md b/apps/opik-documentation/documentation/docs/tracing/integrations/groq.md
index a51899c7ce..984631bc4f 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/groq.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/groq.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Groq
+description: Describes how to track Groq LLM calls using Opik
 ---
 
 # Groq
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/haystack.md b/apps/opik-documentation/documentation/docs/tracing/integrations/haystack.md
index f0808e34f2..6ebfbc7b8d 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/haystack.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/haystack.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Haystack
+description: Describes how to track Haystack pipeline runs using Opik
 ---
 
 # Haystack
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/langchain.md b/apps/opik-documentation/documentation/docs/tracing/integrations/langchain.md
index 7144688601..1567983d00 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/langchain.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/langchain.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: LangChain
+description: Describes how to use Opik with LangChain
 ---
 
 # LangChain
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/langgraph.md b/apps/opik-documentation/documentation/docs/tracing/integrations/langgraph.md
index 4362d0096f..fde8fa5fa9 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/langgraph.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/langgraph.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: LangGraph
+description: Describes how to track LangGraph Agent executions using Opik
 ---
 
 # LangGraph
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/litellm.md b/apps/opik-documentation/documentation/docs/tracing/integrations/litellm.md
index abcbd463e4..0dd477ca40 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/litellm.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/litellm.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: LiteLLM
+description: Describes how to track LiteLLM LLM calls using Opik
 ---
 
 # LiteLLM
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/llama_index.md b/apps/opik-documentation/documentation/docs/tracing/integrations/llama_index.md
index 0964d82dea..ca0973e7e0 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/llama_index.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/llama_index.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: LlamaIndex
+description: Describes how to track LlamaIndex pipelines using Opik
 ---
 
 # LlamaIndex
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/ollama.md b/apps/opik-documentation/documentation/docs/tracing/integrations/ollama.md
index 042b2b898e..c4b9574be8 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/ollama.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/ollama.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Ollama
+description: Describes how to track Ollama LLM calls using Opik
 ---
 
 # Ollama
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/openai.md b/apps/opik-documentation/documentation/docs/tracing/integrations/openai.md
index c4c23a759d..f8efa3684e 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/openai.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/openai.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: OpenAI
+description: Describes how to track OpenAI LLM calls using Opik
 ---
 
 # OpenAI
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/overview.md b/apps/opik-documentation/documentation/docs/tracing/integrations/overview.md
index f98376aba5..8b97a59037 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/overview.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/overview.md
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 1
 sidebar_label: Overview
+description: Describes all the integrations provided by Opik and what each framework can be used for
 ---
 
 # Overview
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/predibase.md b/apps/opik-documentation/documentation/docs/tracing/integrations/predibase.md
index 2a2f072745..bd97df52ae 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/predibase.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/predibase.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Predibase
+description: Describes how to track Predibase LLM calls using Opik
 ---
 
 # Using Opik with Predibase
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/ragas.md b/apps/opik-documentation/documentation/docs/tracing/integrations/ragas.md
index 709be79cd3..48fd243a44 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/ragas.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/ragas.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Ragas
+description: Describes how to log Ragas scores to the Opik platform
 ---
 
 # Ragas
diff --git a/apps/opik-documentation/documentation/docs/tracing/integrations/watsonx.md b/apps/opik-documentation/documentation/docs/tracing/integrations/watsonx.md
index c44c22e202..2a632cbd1b 100644
--- a/apps/opik-documentation/documentation/docs/tracing/integrations/watsonx.md
+++ b/apps/opik-documentation/documentation/docs/tracing/integrations/watsonx.md
@@ -1,5 +1,6 @@
 ---
 sidebar_label: watsonx
+description: Describes how to track watsonx LLM calls using Opik
 ---
 
 # watsonx
diff --git a/apps/opik-documentation/documentation/docs/tracing/log_agents.mdx b/apps/opik-documentation/documentation/docs/tracing/log_agents.mdx
index da304da47e..a25d503f11 100644
--- a/apps/opik-documentation/documentation/docs/tracing/log_agents.mdx
+++ b/apps/opik-documentation/documentation/docs/tracing/log_agents.mdx
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Track Agents
+description: Describes how to track agents using Opik
 toc_min_heading_level: 2
 toc_max_heading_level: 4
 ---
diff --git a/apps/opik-documentation/documentation/docs/tracing/log_distributed_traces.md b/apps/opik-documentation/documentation/docs/tracing/log_distributed_traces.md
index 2c9293ff1b..5003e3cb12 100644
--- a/apps/opik-documentation/documentation/docs/tracing/log_distributed_traces.md
+++ b/apps/opik-documentation/documentation/docs/tracing/log_distributed_traces.md
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 3
 sidebar_label: Log Distributed Traces
+description: Describes how to log distributed traces to the Opik platform
 ---
 
 # Log Distributed Traces
diff --git a/apps/opik-documentation/documentation/docs/tracing/log_multimodal_traces.md b/apps/opik-documentation/documentation/docs/tracing/log_multimodal_traces.md
index f800d9a639..03ba680722 100644
--- a/apps/opik-documentation/documentation/docs/tracing/log_multimodal_traces.md
+++ b/apps/opik-documentation/documentation/docs/tracing/log_multimodal_traces.md
@@ -1,6 +1,6 @@
 ---
-sidebar_position: 2
 sidebar_label: Log Multimodal Traces
+description: Describes how to log and view images in traces to the Opik platform
 toc_min_heading_level: 2
 toc_max_heading_level: 4
 ---
diff --git a/apps/opik-documentation/documentation/docs/tracing/log_traces.mdx b/apps/opik-documentation/documentation/docs/tracing/log_traces.mdx
index b75ec61139..3b699be92f 100644
--- a/apps/opik-documentation/documentation/docs/tracing/log_traces.mdx
+++ b/apps/opik-documentation/documentation/docs/tracing/log_traces.mdx
@@ -1,5 +1,6 @@
 ---
 sidebar_label: Log Traces
+description: Describes how to log LLM calls to the Opik platform using function decorators, integrations or the low level client.
 toc_min_heading_level: 2
 toc_max_heading_level: 4
 ---
@@ -277,7 +278,7 @@ def llm_chain(input_text):
 
 You can learn more about the `opik_context` module in the [opik_context reference docs](https://www.comet.com/docs/opik/python-sdk-reference/opik_context/index.html).
 
-#### Disabling automatic logging
+#### Disabling automatic logging of function input and output
 
 You can use the `capture_input` and `capture_output` parameters of the [`@track`](https://www.comet.com/docs/opik/python-sdk-reference/track.html) decorator to disable the automatic logging of the function input and output:
 
@@ -290,6 +291,16 @@ def llm_chain(input_text):
 
 You can then use the `opik_context` module to manually log the trace and span attributes.
 
+#### Disable all tracing
+
+You can disable the logging of traces and spans using the enviornment variable `OPIK_TRACK_DISABLE`, this will turn off the logging for all function decorators:
+
+```python
+import os
+
+os.environ["OPIK_TRACK_DISABLE"] = "true"
+```
+
 ### Using the low-level Opik client
 
 If you want full control over the data logged to Opik, you can use the [`Opik`](https://www.comet.com/docs/opik/python-sdk-reference/Opik.html) client to log traces, spans, feedback scores and more.
diff --git a/apps/opik-documentation/documentation/docs/tracing/production_monitoring.md b/apps/opik-documentation/documentation/docs/tracing/production_monitoring.md
index 33c7f230fa..3afbdd5d0d 100644
--- a/apps/opik-documentation/documentation/docs/tracing/production_monitoring.md
+++ b/apps/opik-documentation/documentation/docs/tracing/production_monitoring.md
@@ -1,8 +1,6 @@
 ---
-sidebar_position: 1
 sidebar_label: Production Monitoring
-toc_min_heading_level: 2
-toc_max_heading_level: 4
+description: Describes how to monitor your LLM applications in production using Opik
 ---
 
 # Production Monitoring
diff --git a/apps/opik-documentation/documentation/docs/tracing/sdk_configuration.mdx b/apps/opik-documentation/documentation/docs/tracing/sdk_configuration.mdx
index bfcfe80aab..312268f4ef 100644
--- a/apps/opik-documentation/documentation/docs/tracing/sdk_configuration.mdx
+++ b/apps/opik-documentation/documentation/docs/tracing/sdk_configuration.mdx
@@ -1,8 +1,6 @@
 ---
-sidebar_position: 6
 sidebar_label: Python SDK Configuration
-toc_min_heading_level: 2
-toc_max_heading_level: 4
+description: Describes how to configure the Python SDK
 ---
 
 import Tabs from "@theme/Tabs";
@@ -114,6 +112,7 @@ Here is a list of the configuration values that you can set:
 | api_key                    | `OPIK_API_KEY`               | The API key to use - Only required if you are using the Opik Cloud version of the platform   |
 | workspace                  | `OPIK_WORKSPACE`             | The workspace to use - Only required if you are using the Opik Cloud version of the platform |
 | project_name               | `OPIK_PROJECT_NAME`          | The project name to use                                                                      |
+| opik_track_disable         | `OPIK_TRACK_DISABLE`         | Flag to disable the tracking of traces and spans - Defaults to `false`                       |
 | default_flush_timeout      | `OPIK_DEFAULT_FLUSH_TIMEOUT` | The default flush timeout to use - Defaults to no timeout                                    |
 | opik_check_tls_certificate | `OPIK_CHECK_TLS_CERTIFICATE` | Flag to check the TLS certificate of the Opik server - Defaults to `true`                    |
 
diff --git a/apps/opik-documentation/documentation/docusaurus.config.ts b/apps/opik-documentation/documentation/docusaurus.config.ts
index 4041c046fa..f1ad5c1fa8 100644
--- a/apps/opik-documentation/documentation/docusaurus.config.ts
+++ b/apps/opik-documentation/documentation/docusaurus.config.ts
@@ -45,7 +45,6 @@ const config: Config = {
       "classic",
       {
         docs: {
-          // Remove the sidebarPath option
           sidebarPath: "./sidebars.ts",
           routeBasePath: "/",
           docItemComponent: "@theme/ApiItem",
@@ -60,6 +59,15 @@ const config: Config = {
 
   plugins: [
     "docusaurus-plugin-sass",
+    [
+      "docusaurus-plugin-llms-txt",
+      {
+        title: "Opik documentation",
+        description:
+          "Opik is an open source LLM evaluation platform that includes a prompt playground, automated evaluation metrics, and a LLM gateway.",
+        fullLLMsTxt: true,
+      },
+    ],
     [
       require.resolve("docusaurus-plugin-search-local"),
       {
@@ -110,9 +118,10 @@ const config: Config = {
       title: "Comet Opik",
       items: [
         {
-          type: "doc",
+          type: "docSidebar",
           to: "/",
           label: "Guides",
+          sidebarId: "guide_sidebar",
           position: "left",
           docId: "home",
         },
diff --git a/apps/opik-documentation/documentation/package-lock.json b/apps/opik-documentation/documentation/package-lock.json
index 46d4f486db..163f10e01b 100644
--- a/apps/opik-documentation/documentation/package-lock.json
+++ b/apps/opik-documentation/documentation/package-lock.json
@@ -14,6 +14,7 @@
         "@mdx-js/react": "^3.0.0",
         "@segment/analytics-next": "^1.76.0",
         "clsx": "^2.0.0",
+        "docusaurus-plugin-llms-txt": "^0.1.3",
         "docusaurus-plugin-openapi-docs": "^4.0.1",
         "docusaurus-plugin-sass": "^0.2.5",
         "docusaurus-plugin-search-local": "^2.0.1",
@@ -7309,6 +7310,23 @@
         "node": ">=6"
       }
     },
+    "node_modules/docusaurus-plugin-llms-txt": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/docusaurus-plugin-llms-txt/-/docusaurus-plugin-llms-txt-0.1.3.tgz",
+      "integrity": "sha512-X/Dqf1/soHFjvELWXtfa9C023498HsWWwNPdVZEu0OoznI2Gzf+VCB/4obwpBSV9OeDeI8D94/dK/ETq534k9A==",
+      "license": "MIT",
+      "dependencies": {
+        "fs-extra": "^11.2.0",
+        "gray-matter": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=16.14"
+      },
+      "peerDependencies": {
+        "@docusaurus/core": ">=2.0.0",
+        "react": ">=16.8.0"
+      }
+    },
     "node_modules/docusaurus-plugin-openapi-docs": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/docusaurus-plugin-openapi-docs/-/docusaurus-plugin-openapi-docs-4.2.0.tgz",
diff --git a/apps/opik-documentation/documentation/package.json b/apps/opik-documentation/documentation/package.json
index d725ce5b49..637b3465a4 100644
--- a/apps/opik-documentation/documentation/package.json
+++ b/apps/opik-documentation/documentation/package.json
@@ -22,6 +22,7 @@
     "@mdx-js/react": "^3.0.0",
     "@segment/analytics-next": "^1.76.0",
     "clsx": "^2.0.0",
+    "docusaurus-plugin-llms-txt": "^0.1.3",
     "docusaurus-plugin-openapi-docs": "^4.0.1",
     "docusaurus-plugin-sass": "^0.2.5",
     "docusaurus-plugin-search-local": "^2.0.1",
diff --git a/apps/opik-documentation/documentation/sidebars.ts b/apps/opik-documentation/documentation/sidebars.ts
index b140080569..4c42861003 100644
--- a/apps/opik-documentation/documentation/sidebars.ts
+++ b/apps/opik-documentation/documentation/sidebars.ts
@@ -12,7 +12,7 @@ import apiSidebar from "./docs/reference/rest_api/sidebar";
  Create as many sidebars as you want.
  */
 const sidebars: SidebarsConfig = {
-  guideSidebar: [
+  guide_sidebar: [
     "home",
     "quickstart",
     {