Fix pdf template for latest datatype refactorings

Author: blythed

.gitignore

@@ -197,3 +197,4 @@ test/sleep.json
 /deploy/testenv/requirements.txt
 /superduper/rest/superdupertmp
 /example*
+.cache

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 **Before you create a Pull Request, remember to update the Changelog with your changes.**
 
-## Changes Since Last Release 
+## Changes Since Last Release   
 
 #### Changed defaults / behaviours 
 

plugins/sentence_transformers/superduper_sentence_transformers/model.py

@@ -2,7 +2,6 @@
 
 from sentence_transformers import SentenceTransformer as _SentenceTransformer
 from superduper.backends.query_dataset import QueryDataset
-from superduper.base.enums import DBType
 from superduper.components.component import ensure_initialized
 from superduper.components.model import Model, Signature, _DeviceManaged
 
@@ -62,9 +61,9 @@ def __post_init__(self, db, example):
             self.object = _SentenceTransformer(self.model, device=self.device)
             self._default_model = True
 
-    def dict(self, metadata: bool = True, defaults: bool = True):
+    def dict(self, metadata: bool = True, defaults: bool = True, refs: bool = False):
         """Serialize as a dictionary."""
-        r = super().dict(metadata=metadata, defaults=defaults)
+        r = super().dict(metadata=metadata, defaults=defaults, refs=refs)
         if self._default_model:
             del r['object']
         return r
@@ -125,4 +124,3 @@ def _pre_create(self, db):
         """
         if self.datatype is not None:
             return
-

templates/pdf_rag/Dockerfile

@@ -2,15 +2,13 @@ FROM python:3.10
 
 WORKDIR /app
 
-COPY ./superduper .
-COPY ./plugins .
+COPY ./superduper superduper
+COPY pyproject.toml pyproject.toml
+COPY ./plugins plugins
 COPY ./templates/pdf_rag templates/pdf_rag
 
-RUN pip install -e . && pip install plugins/mongodb && pip install streamlit
-RUN pip install --no-cache-dir ./templates/pdf_rag/requirements.txt
-
-RUN superduper bootstrap ./templates/pdf_rag
-RUN python3 add_data.py
-RUN superduper apply pdf_rag
-
-CMD ["streamlit", "run", "templates/pdf_rag/streamlit.py", "--server.runOnSave", "true"]
+RUN pip install . plugins/mongodb streamlit ipython jupyter -r ./templates/pdf_rag/requirements.txt
+RUN chmod +x templates/pdf_rag/install.sh
+RUN ./templates/pdf_rag/install.sh
+    
+CMD ["/bin/bash"]

templates/pdf_rag/README.md

@@ -0,0 +1,68 @@
+## PDF RAG template
+
+**Clone the code**
+
+```bash
+git clone --branch main --depth 1 [email protected]:superduper-io/superduper.git
+```
+
+**Build the docker**
+
+```bash
+docker build . -f templates/pdf_rag/Dockerfile -t pdf-rag
+```
+
+**Start the docker container with the ports opened and data mounted**
+
+You need to mount a local directory to `/app/data` in the container.
+
+***The PDF directory should be a sub-directory of this directory!***
+
+```bash
+docker run -it -p 8501:8501 -v ~/.superduper/:/root/.superduper -v ./data/:/app/data pdf-rag bash
+```
+
+**Set your OpenAI or Ollama credentials**
+
+(Run this in the container)
+
+```bash
+export OPENAI_API_KEY=ollama  # or sk-<secret> for openai
+export OPENAI_BASE_URL=...    # URL of ollama server if applicable
+```
+
+**Set your MongoDB connection**
+
+```bash
+export SUPERDUPER_DATA_BACKEND=mongodb://host.docker.internal:27017/test_db
+export SUPERDUPER_ARTIFACT_STORE=filesystem://./data
+```
+
+***Prepare the app with your choice of models and data**
+
+```bash
+bash templates/pdf_rag/start.sh bodybuilder <embedding_model> <llm_model>
+```
+
+For example, on OpenAI:
+
+```bash
+bash templates/pdf_rag/start.sh bodybuilder text-embedding-ada-002 gpt-3.5-turbo
+```
+
+For example, on Ollama:
+
+```bash
+bash templates/pdf_rag/start.sh bodybuilder nomic-embed-text:latest llama3.1:70b
+```
+
+***Run the app's frontend**
+
+```bash
+python3 -m streamlit run templates/pdf-rag/streamlit.py
+```
+
+### Notes
+
+- If you exit the docker container, you will need to reset your environment variables (`export $...`)
+- If you are using Ollama, you need to use models which are installed on the server

templates/pdf_rag/add_data.py

@@ -0,0 +1,18 @@
+import os
+import sys
+
+from superduper import Schema, Table
+from superduper.components.datatype import file
+
+
+pdf_folder = sys.argv[1]
+pdf_names = [pdf for pdf in os.listdir(pdf_folder) if pdf.endswith(".pdf")]
+pdf_paths = [os.path.join(pdf_folder, pdf) for pdf in pdf_names]
+data = [{"url": pdf_path, "file": pdf_path} for pdf_path in pdf_paths]
+from superduper import superduper
+db = superduper()
+COLLECTION_NAME = next(x for x in pdf_folder.split('/')[::-1] if x)
+schema = Schema(identifier="myschema", fields={'url': 'str', 'file': file})
+table = Table(identifier=COLLECTION_NAME, schema=schema)
+db.apply(table, force=True)
+db[COLLECTION_NAME].insert(data).execute()