Auto-build FAISS index on startup when absent

README.md

@@ -0,0 +1,50 @@
+# Evaluation Policy Q&A Assistant
+
+A Streamlit RAG (Retrieval-Augmented Generation) chatbot for querying Canada's *Policy on Results* (Treasury Board, 2016) in English and French.
+
+## How it works
+
+1. Source PDFs in `data/` are indexed into a FAISS vector store via `ingest.py`.
+2. At runtime, `app/main.py` loads the FAISS index and uses [LangChain](https://github.com/langchain-ai/langchain) + [Groq](https://groq.com/) to answer questions.
+
+### Self-healing startup
+
+The app automatically builds the FAISS index on first launch if `data/faiss_index/` is absent.  
+This means **no manual `python ingest.py` step is required** when deploying to Streamlit Cloud or any fresh environment that only has the repository files.
+
+A spinner is shown while the index is being created (~1 minute on first run).  
+Once built, the index is cached and subsequent startups load it instantly.
+
+> **Note:** `data/faiss_index/` is listed in `.gitignore` because it is a generated artifact.  
+> The source PDFs committed to `data/` are sufficient to reconstruct it.
+
+## Local setup
+
+```bash
+# 1. Install dependencies
+pip install -r requirements.txt
+
+# 2. Add your Groq API key
+echo "GROQ_API_KEY=<your-key>" > .env
+
+# 3. (Optional) Pre-build the index manually
+python ingest.py
+
+# 4. Run the app
+streamlit run app/main.py
+```
+
+If you skip step 3, the app will build the index automatically on first launch.
+
+## Deployment (Streamlit Cloud)
+
+1. Fork / push this repository to GitHub.
+2. Create a new app on [share.streamlit.io](https://share.streamlit.io), pointing to `app/main.py`.
+3. Add `GROQ_API_KEY` as a secret in the app settings.
+4. Deploy — the index is built automatically on the first cold start.
+
+## Environment
+
+| Variable | Description |
+|---|---|
+| `GROQ_API_KEY` | API key for the Groq LLM (required) |

app/main.py

@@ -2,6 +2,7 @@
 os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
+import sys
 from pathlib import Path
 from dotenv import load_dotenv
 import streamlit as st
@@ -10,7 +11,17 @@
 from langchain_groq import ChatGroq
 
 load_dotenv()
-INDEX_DIR = Path("data/faiss_index")
+
+# Resolve paths relative to the repository root so the app works correctly
+# regardless of the working directory at startup (e.g. Streamlit Cloud).
+_REPO_ROOT = Path(__file__).parent.parent
+DATA_DIR = _REPO_ROOT / "data"
+INDEX_DIR = DATA_DIR / "faiss_index"
+
+# Make sure the repo root is on sys.path so that ingest.py can be imported.
+_repo_root_str = str(_REPO_ROOT)
+if _repo_root_str not in sys.path:
+    sys.path.insert(0, _repo_root_str)
 
 st.set_page_config(page_title="Evaluation Policy Assistant", page_icon="🇨🇦", layout="wide")
 st.title("Evaluation Policy Q&A Assistant")
@@ -22,13 +33,20 @@
 
 @st.cache_resource(show_spinner="Loading knowledge base...")
 def load_vectorstore():
+    if not INDEX_DIR.exists():
+        with st.spinner("Knowledge base not found -- building it now from source PDFs (this takes a minute on first run)..."):
+            try:
+                from ingest import build_index
+                build_index(data_dir=DATA_DIR, index_dir=INDEX_DIR)
+            except Exception as exc:
+                st.error(
+                    f"Failed to build the knowledge base automatically: {exc}\n\n"
+                    "Please run `python ingest.py` from the repository root and redeploy."
+                )
+                st.stop()
     emb = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
     return FAISS.load_local(str(INDEX_DIR), emb, allow_dangerous_deserialization=True)
 
-if not INDEX_DIR.exists():
-    st.error("Index not found. Run python ingest.py first.")
-    st.stop()
-
 vs = load_vectorstore()
 llm = ChatGroq(model="llama-3.3-70b-versatile", temperature=0.1, api_key=os.getenv("GROQ_API_KEY"))
 

ingest.py

@@ -4,33 +4,49 @@
 from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
 
-DATA_DIR = Path("data")
+_REPO_ROOT = Path(__file__).parent
+DATA_DIR = _REPO_ROOT / "data"
 INDEX_DIR = DATA_DIR / "faiss_index"
 
 DOCS = {
     "policy_on_results_EN.pdf": "en",
     "policy_on_results_FR.pdf": "fr",
 }
 
-all_docs = []
-# Smaller chunks for more precise retrieval
-splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=50)
-
-for fname, lang in DOCS.items():
-    fpath = str(DATA_DIR / fname)
-    print(f"Loading {fname}...")
-    loader = PyPDFLoader(fpath)
-    pages = loader.load()
-    for p in pages:
-        p.metadata["language"] = lang
-        p.metadata["source"] = fname
-    chunks = splitter.split_documents(pages)
-    all_docs.extend(chunks)
-    print(f"  -> {len(chunks)} chunks")
-
-print(f"\nTotal chunks: {len(all_docs)}")
-print("Building FAISS index...")
-embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-vectorstore = FAISS.from_documents(all_docs, embeddings)
-vectorstore.save_local(str(INDEX_DIR))
-print(f"Done! Index saved at {INDEX_DIR}")
+
+def build_index(data_dir: Path = DATA_DIR, index_dir: Path = INDEX_DIR) -> None:
+    """Build the FAISS vector index from the source PDFs and save it to *index_dir*.
+
+    Parameters
+    ----------
+    data_dir:
+        Directory that contains the source PDF files.
+    index_dir:
+        Directory where the FAISS index will be saved.
+    """
+    all_docs = []
+    # Smaller chunks for more precise retrieval
+    splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=50)
+
+    for fname, lang in DOCS.items():
+        fpath = str(data_dir / fname)
+        print(f"Loading {fname}...")
+        loader = PyPDFLoader(fpath)
+        pages = loader.load()
+        for p in pages:
+            p.metadata["language"] = lang
+            p.metadata["source"] = fname
+        chunks = splitter.split_documents(pages)
+        all_docs.extend(chunks)
+        print(f"  -> {len(chunks)} chunks")
+
+    print(f"\nTotal chunks: {len(all_docs)}")
+    print("Building FAISS index...")
+    embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    vectorstore = FAISS.from_documents(all_docs, embeddings)
+    vectorstore.save_local(str(index_dir))
+    print(f"Done! Index saved at {index_dir}")
+
+
+if __name__ == "__main__":
+    build_index()